; docformat = 'rst'
;+
; :Author:
; Don Woodraska
;
; :Copyright:
; Copyright 2015 The Regents of the University of Colorado.
; All rights reserved. This software was developed at the
; University of Colorado's Laboratory for Atmospheric and
; Space Physics, SDO EVE Science Processing Operations Center.
;
; :Version:
; $Id: eve_download_data.pro 66720 2015-04-01 17:52:40Z dlwoodra $
;
;-
;+
; This function downloads all SDO EVE data products of a specified
; type for the specified 7-digit year and day of year. The default
; data product type is the EVE Level 2 Spectrum (/evS_L2). Other
; options include /evL_L2, /esp_L1, /level3, /high_res_merged,
; /angstrom_merged, and /nanometer_merged.
;
; All EVE data products from level 1, 2, and 3 are calibrated.
;
; This software requires IDL 6.4 or greater.
;
; :Params:
; yyyydoy: in, required, type=long or lonarr
; 7-digit year and day of year (2010365)
;
; :Keywords:
; evS_L2: in, optional, type=boolean
; Set this keyword to download FITS files matching EVS_L2_*fit.gz
; These are the routine level 2 spectra products. They contain high
; resolution, high cadence spectra files.
; This is the DEFAULT option if no other is specified.
; evL_L2: in, optional, type=boolean
; Set this keyword to download FITS files matching EVL_L2_*fit.gz
; These are the routine level 2 extracted lines, integrated
; bands, and broadband diodes integrated to match the 10-second
; spectrum cadence. These data have high cadence lines, bands,
; and diodes.
; esp_L1: in, optional, type=boolean
; Set this keyword to download FITS files matching esp_L1_* These
; are the routine level 1 ESP products at native 4 Hz integration
; rates. These are highest cadence ESP broadband diode data
; files.
; level3: in, optional, type=boolean
; Set this keyword to download FITS files matching EVE_L3_*
; These are the routine level 3 data products and contain the
; daily averages of EVE measurements. These daily data files
; contain the high resolution spectrum plus daily averages of all
; the lines, bands, and diodes from the level 2 products.
; high_res_merged: in, optional, type=boolean
; Set this keyword to download the current mission merged SAV file
; This contains the highest resolution daily average spectra from
; level 3 merged into one mission-length file. FITS and NetCDF
; versions are also available from the web site.
; angstrom_merged: in, optional, type=boolean
; Set this keyword to download the current mission merged SAV file
; These are daily averages from the high_res_merged integrated
; onto 1 angstrom bins. FITS and NetCDF versions are also
; available from the web site.
; nanometer_merged: in, optional, type=boolean
; Set this keyword to download the current mission merged SAV file
; These are daily averages from the angstrom_merged integrated
; onto 1 nanometer bins. FITS and NetCDF versions are also
; available from the web site.
; output_directory: in, optional, type=string
; Provide a string indicating where the data files will be
; placed on your computer. If the direcory does not exist, a
; prompt will be issued to create the directory.
;
; :Examples:
; Normal usage would be to define an array of 7-digit YYYYDOY for
; the individual dates to request.::
; IDL> ydlist = long(2015000) + [1, 2, 3]
; IDL> status = eve_download_data( ydlist, /evs_L2 )
;
; Normally, you would want to place the data files into a directory
; for later analysis.::
; IDL> mydir = './data'
; IDL> status = eve_download_data( 2015001, /evs_L2, output_directory=mydir )
;
; Any routine EVE data products can be downloaded for the list of
; days provided.
; To download the level 2 lines/bands/diodes files::
; IDL> status = eve_download_data( yyyydoy, /evl_L2 )
;
; To download the level 3 daily average files (one per day)::
; IDL> status = eve_download_data( yyyydoy, /level3 )
;
; To download the ESP level 1 4 Hz diode measurements::
; IDL> status = eve_download_data( yyyydoy, /esp_L1)
;
; To download the high resolution mission merged IDL SAV file::
; IDL> status = eve_download_data( /high_res_merged )
;
; To download the 1 angstrom integrated mission merged IDL SAV file::
; IDL> status = eve_download_data( /angstrom_merged )
;
; To download the 1 nm integrated mission merged IDL SAV file::
; IDL> status = eve_download_data( /nanometer_merged )
;
; Note that only one product type can be downloaded with one call
; to this function. Specifying multiple product types will only
; download one, the last one that is checked. Consider them
; mutually exclusive.
;
;-
function eve_download_data, yyyydoy, $
evS_L2=evS_L2, $
evL_L2=evL_L2, $
esp_L1=esp_L1, $
level3=level3, $
high_res_merged=high_res_merged, $
angstrom_merged=angstrom_merged, $
nanometer_merged=nanometer_merged, $
output_directory=output_directory
if float(!version.release) lt 6.4 then begin
print,'ERROR: eve_download_data - required IDL version 6.4 or greater'
print,'This version of IDL is '+!version.release
return,-1
endif
produrl = 'http://lasp.colorado.edu/eve/data_access/evewebdataproducts/'
; if output_directory was not passed, set it to the current directory
if size(output_directory,/type) eq 0 then begin
; the only way to get the current directory is to try to change it
;cd,current=output_directory, '.'+path_sep()
level = 'level3'
if size(esp_L1, /type) NE 0 THEN level = 'level1'
if size(evS_L2, /type) NE 0 THEN level = 'level2'
if size(evL_L2, /type) NE 0 THEN level = 'level2'
year = FIX( yyyydoy / 1000L )
output_directory = string(format='(%"%s%1s%s%1s%04d%1s")',getenv('EVE_DATA'), path_sep(), level, path_sep(), year, path_sep())
PRINT, "Saving data to: " + output_directory
dirInfo = FILE_INFO( output_directory )
IF dirInfo.EXISTS EQ 0 THEN FILE_MKDIR, output_directory
endif
; append a path separator if needed
if strmid(output_directory,strlen(output_directory)-1,1) ne path_sep() then $
output_directory += path_sep()
; create local directory if it does not exist
if file_test(output_directory,/directory,/write) eq 0 then begin
result = dialog_message('The output directory does not exist. Create directory?',/question)
if strmatch(result,'Yes') eq 0 then begin
print,'ERROR: eve_download_data - User aborted. Output directory not created.'
return,-1
endif
file_mkdir, output_directory
endif
if size(yyyydoy,/type) eq 0 and $
(keyword_set(high_res_merged) eq 0 and $
keyword_set(angstrom_merged) eq 0 and $
keyword_set(nanometer_merged) eq 0) then begin
print,'ERROR: eve_download_data - no date specified. This function requires a yyyydoy formed date, or an array of yyyydoy dates'
return,-1
endif
if keyword_set(evS_L2) eq 0 and $
keyword_set(evL_L2) eq 0 and $
keyword_set(esp_l1) eq 0 and $
keyword_set(level3) eq 0 and $
keyword_set(high_res_merged) eq 0 and $
keyword_set(angstrom_merged) eq 0 and $
keyword_set(nanometer_merged) eq 0 then begin
print,'WARNING: eve_download_data - no product specified, assuming level 2 spectrum products are needed'
evs_l2=1 ; define a default selected option
endif
;
; loop over each yyyydoy element
;
for i=0L,n_elements(yyyydoy)-1 do begin
if yyyydoy[i] lt 2010120 then begin
if keyword_set(high_res_merged) eq 0 and $
keyword_set(angstrom_merged) eq 0 and $
keyword_set(nanometer_merged) eq 0 then begin
print,'ERROR: eve_download_data - received yyyydoy < 2010120'
print,'EVE normal operations began 2010120, setting to 2010120'
endif
; always prevent invalid dates
; do it quietly for merged files since it does not matter for those
yyyydoy[i]=2010120L
endif
stryyyy = string(yyyydoy[i] / 1000L, form='(i4.4)') ; 2010-...
strdoy = string(yyyydoy[i] mod 1000L, form='(i3.3)') ; 001-366
stryyyydoy = stryyyy + strdoy
if keyword_set(evS_L2) then begin
urlbase = produrl + 'level2/' ; yyyy + ddd + '/EVS_L2_yyyydoy_hh_vvv_rr.fit.gz'
urlbase = urlbase + stryyyy + '/' + strdoy + '/'
filepattmatch = '*EVS_L2_' + stryyyydoy + '*.fit.gz*'
fileregexmatch = 'EVS_L2_' + stryyyydoy + '_[0-2][0-9]_[0-9]{3}_[0-9]{2}\.fit\.gz'
endif
if keyword_set(evL_L2) then begin
urlbase = produrl + 'level2/' ; yyyy + ddd + '/EVL_L2_yyyydoy_hh_vvv_rr.fit.gz'
urlbase = urlbase + stryyyy + '/' + strdoy + '/'
filepattmatch = '*EVL_L2_' + stryyyydoy + '*.fit.gz*'
fileregexmatch = 'EVL_L2_' + stryyyydoy + '_[0-2][0-9]_[0-9]{3}_[0-9]{2}\.fit\.gz'
endif
if keyword_set(esp_L1) then begin
urlbase = produrl + 'level1/esp/' ; yyyy + '/esp_L1_yyyydoy_vvv.fit'
urlbase = urlbase + stryyyy + '/'
filepattmatch = '*esp_L1_' + stryyyydoy + '*.fit*'
fileregexmatch = 'esp_L1_' + stryyyydoy + '_[0-9]{3}\.fit'
endif
if keyword_set(level3) then begin
urlbase = produrl + 'level3/' ; yyyy +'/EVE_L3_yyyydoy_vvv.fit'
urlbase = urlbase + stryyyy + '/'
filepattmatch = '*EVE_L3_' + stryyyydoy + '*.fit*'
fileregexmatch = 'EVE_L3_' + stryyyydoy + '_[0-9]{3}_[0-9]{2}\.fit'
endif
if keyword_set(high_res_merged) then begin
urlbase = produrl + 'merged/' ; '/EVE_L3_merged_yyyydoy_vvv.sav'
filepattmatch = '*EVE_L3_merged_???????_???.sav*'
fileregexmatch = 'EVE_L3_merged_[0-9]{7}_[0-9]{3}\.sav'
endif
if keyword_set(angstrom_merged) then begin
urlbase = produrl + 'merged/' ; '/EVE_L3_merged_yyyydoy_vvv.sav'
filepattmatch = '*EVE_L3_merged_1a_???????_???.sav*'
fileregexmatch = 'EVE_L3_merged_1a_[0-9]{7}_[0-9]{3}\.sav'
endif
if keyword_set(nanometer_merged) then begin
urlbase = produrl + 'merged/' ; '/EVE_L3_merged_yyyydoy_vvv.sav'
filepattmatch = '*EVE_L3_merged_1nm_???????_???.sav*'
fileregexmatch = 'EVE_L3_merged_1nm_[0-9]{7}_[0-9]{3}\.sav'
endif
;
; Gracefully retry if any error occurs with catch
;
CATCH, errorStatus
if (errorStatus ne 0) then begin
;CATCH,/CANCEL ; disable future error handling
; get whatever info you can
oUrl->GetProperty, RESPONSE_CODE=response, RESPONSE_HEADER=rspHdr
; display the error info to the user
; note that 200 means OK connection, any 2xx code is success
; 4xx codes mean there is a client error of some kind
; 5xx codes mean there is a server error
print,''
print,'ERROR: eve_download_data - code=',strtrim(response,2)
print,'ERROR: eve_download_data - rspHdr=',rspHdr
; cleanup
OBJ_DESTROY, oUrl
; ask to try again
result = dialog_message('The EVE file download did not work. Try again?',/question)
if strmatch(result,'Yes') eq 0 then begin
print,'ERROR: eve_download_data -aborted by user request'
return,-1
endif
endif
;
; try to connect to URL to get latest merged file
;
oUrl = OBJ_NEW('IDLnetUrl')
; try to get directory listing
thehtml = oUrl->Get(url=urlbase,/string_array)
; parse the html for the latest save file
htmlline = where(strmatch(thehtml,filepattmatch) eq 1, n_files )
if n_files eq 0 then begin
print,'ERROR: eve_download_data - no product file found on EVE web site'
stop
endif
;
; extract the returned filename string buried in the html line
;
for j=0L,n_files-1 do begin
thefile = stregex( thehtml[htmlline[j]], fileregexmatch, /extract )
url = urlbase + thefile
localfile = output_directory+thefile
; attempt to download one file
print,'Downloading EVE data product file -> ',url
result = oUrl->Get( URL=url, file=localfile )
endfor
; cleanup
oUrl->CloseConnections
OBJ_DESTROY, oUrl
endfor
;
; return
;
return, 1
end