IDM下载ASF文件

第一次写文章，界面不太好看，还请见谅。
#######################################################################
虽然ASF网站提供了python批下载的脚本，但是不知道是由于网络原因还是我的python脚本不对，下载经常中断，有时候甚至运行了一天一个文件也没有下下来，一个一个的手动下载又太麻烦。所以使用IDM下载器进行批量下载。
这里面的第一个问题是url链接，我们直接输入python脚本中的链接是无法运行的，需要修改download_file_with_cookiejar函数，我修改后的代码如下：（将test打印的列表放到IDM）

 def download_file_with_cookiejar(self, url, file_count, total, recursion=False):
       test=[]
       # see if we've already download this file and if it is that it is the correct size
       download_file = os.path.basename(url).split('?')[0]
       if os.path.isfile(download_file):
          try:
             request = Request(url)
             request.get_method = lambda : 'HEAD'
             response = urlopen(request, timeout=30)
             remote_size = self.get_total_size(response)
             # Check that we were able to derive a size.
             if remote_size:
                 local_size = os.path.getsize(download_file)
                 if remote_size < (local_size+(local_size*.01)) and remote_size > (local_size-(local_size*.01)):
                     print (" > Download file {0} exists! \n > Skipping download of {1}. ".format(download_file, url))
                     return None,None
                 #partial file size wasn't full file size, lets blow away the chunk and start again
                 print (" > Found {0} but it wasn't fully downloaded. Removing file and downloading again.".format(download_file))
                 os.remove(download_file)

          except ssl.CertificateError as e:
             print (" > ERROR: {0}".format(e))
             print (" > Could not validate SSL Cert. You may be able to overcome this using the --insecure flag")
             return False,None

          except HTTPError as e:
             if e.code == 401:
                 print (" > IMPORTANT: Your user may not have permission to download this type of data!")
             else:
                 print (" > Unknown Error, Could not get file HEAD: {0}".format(e))

          except URLError as e:
             print ("URL Error (from HEAD): {0}, {1}".format( e.reason, url))
             if "ssl.c" in "{0}".format(e.reason):
                 print ("IMPORTANT: Remote location may not be accepting your SSL configuration. This is a terminal error.")
             return False,None
       for url in self.files:
       # attempt https connection
           try:
              request = Request(url)
              response = urlopen(request, timeout=30)

          # Watch for redirect
              if response.geturl() != url:

                 # See if we were redirect BACK to URS for re-auth.
                 if 'https://urs.earthdata.nasa.gov/oauth/authorize' in response.geturl():

                     if recursion:
                         print (" > Entering seemingly endless auth loop. Aborting. ")
                         return False, None

                     # make this easier. If there is no app_type=401, add it
                     new_auth_url = response.geturl()
                     print("lalalalal")
                     print(new_auth_url)
                     if "app_type" not in new_auth_url:
                         new_auth_url += "&app_type=401"

                     print (" > While attempting to download {0}....".format(url))
                     print (" > Need to obtain new cookie from {0}".format(new_auth_url))
                     old_cookies = [cookie.name for cookie in self.cookie_jar]
                     opener = build_opener(HTTPCookieProcessor(self.cookie_jar), HTTPHandler(), HTTPSHandler(**self.context))
                     request = Request(new_auth_url)
                     try:
                         response = opener.open(request)
                         for cookie in self.cookie_jar:
                             if cookie.name not in old_cookies:
                                  print (" > Saved new cookie: {0}".format(cookie.name))

                                  # A little hack to save session cookies
                                  if cookie.discard:
                                       cookie.expires = int(time.time()) + 60*60*24*30
                                       print (" > Saving session Cookie that should have been discarded! ")

                         #self.cookie_jar.save(self.cookie_jar_path, ignore_discard=True, ignore_expires=True)
                     except HTTPError as e:
                         print ("HTTP Error: {0}, {1}".format( e.code, url))
                         return False,None

                     # Okay, now we have more cookies! Lets try again, recursively!
                     print (" > Attempting download again with new cookies!")

                     return self.download_file_with_cookiejar(url, file_count, total, recursion=True)
                 print (" > 'Temporary' Redirect download @ Remote archive:\n > {0}".format(response.geturl()))
                 test.append(response.geturl())
                 print(test)

          # seems to be working
              #print ("({0}/{1}) Downloading {2}".format(file_count, total, url))

              # Open our local file for writing and build status bar
              # tf = tempfile.NamedTemporaryFile(mode='w+b', delete=False, dir='.')
              # self.chunk_read(response, tf, report_hook=self.chunk_report)
              #
              # # Reset download status
              # sys.stdout.write('\n')
              #
              # tempfile_name = tf.name
              # tf.close()

       #handle errors
           except HTTPError as e:
              # print ("HTTP Error: {0}, {1}".format( e.code, url))
              #
              # if e.code == 401:
              #    print (" > IMPORTANT: Your user does not have permission to download this type of data!")
              #
              # if e.code == 403:
              #    print (" > Got a 403 Error trying to download this file.  ")
              #    print (" > You MAY need to log in this app and agree to a EULA. ")
              #
              return

然后就可以啦！！
新人求关注！