diff --git a/create_report.py b/create_report.py index d3e7c23..4cc6f3c 100644 --- a/create_report.py +++ b/create_report.py @@ -102,7 +102,7 @@ def getListOfFiles(dirName): # create a list of file and sub directories # names in the given directory listOfFile = os.listdir(dirName) - allFiles = list() + allFiles = [] # Iterate over all the entries for entry in listOfFile: # Create full path @@ -112,7 +112,7 @@ def getListOfFiles(dirName): allFiles = allFiles + getListOfFiles(fullPath) else: allFiles.append(fullPath) - + return allFiles @@ -121,17 +121,16 @@ def getListOfMetadataFiles(dirName): # create a list of file and sub directories # names in the given directory listOfFile = os.listdir(dirName) - allFiles = list() + allFiles = [] # Iterate over all the entries for entry in listOfFile: # Create full path fullPath = os.path.join(dirName, entry) - # If entry is a directory then get the list of files in this directory + # If entry is a directory then get the list of files in this directory if os.path.isdir(fullPath): allFiles = allFiles + getListOfFiles(fullPath) - else: - if regex.match(fullPath): - allFiles.append(fullPath) + elif regex.match(fullPath): + allFiles.append(fullPath) #filtered = [i for i in full if not regex.match(i)] #return filtered diff --git a/order_api.py b/order_api.py index 6e6ba3b..ba19d32 100644 --- a/order_api.py +++ b/order_api.py @@ -56,7 +56,7 @@ # handle globs of *.geojson def get_remaining_quota(): - leftquota = 0.0 + leftquota = 0.0 main = requests.get( 'https://api.planet.com/auth/v1/' + 'experimental/public/my/subscriptions', @@ -65,35 +65,35 @@ def get_remaining_quota(): if main.status_code == 200: content = main.json() for item_id in content: - if (item_id['quota_sqkm'])is not None: - leftquota = (float(item_id['quota_sqkm'] - float(item_id['quota_used']))) - print('Remaining Quota in SqKm: %s' % leftquota) - return(leftquota) - else: - print('No Quota Allocated') - + if (item_id['quota_sqkm'])is not None: + leftquota = (float(item_id['quota_sqkm'] - float(item_id['quota_used']))) + print(f'Remaining Quota in SqKm: {leftquota}') + return(leftquota) + else: + print('No Quota Allocated') + else: - print('Failed with exception code: ' + str(main.status_code)) + print(f'Failed with exception code: {str(main.status_code)}') def calculate_sqkm(df,kbuf): pll = df[['lat','long']] geometryX = [Point(xy) for xy in zip(pll['long'], pll['lat'])] - + lbuf = 0.0025#0.01 xbuf = lbuf#/2.0 mn = 0 totalm2 = 0.0 for pp in geometryX: - listarray = [] - #listarray.append([pp.x, pp.y]) - listarray.append([pp.x+xbuf, pp.y-lbuf]) - listarray.append([pp.x-xbuf, pp.y-lbuf]) - listarray.append([pp.x-xbuf, pp.y+lbuf]) - listarray.append([pp.x+xbuf, pp.y+lbuf]) # self intersection? - + listarray = [ + [pp.x + xbuf, pp.y - lbuf], + [pp.x - xbuf, pp.y - lbuf], + [pp.x - xbuf, pp.y + lbuf], + [pp.x + xbuf, pp.y + lbuf], + ] + nparray = np.array(listarray) poly = geometry.Polygon(nparray) - + geom = poly geom_area = ops.transform( partial( @@ -104,7 +104,7 @@ def calculate_sqkm(df,kbuf): lat1=geom.bounds[1], lat2=geom.bounds[3])), geom) - + # Print the area in m^2 #print(geom_area.area) totalm2 += geom_area.area @@ -114,23 +114,22 @@ def calculate_sqkm(df,kbuf): def generate_geojson_from_coords_df(df,kbuf): pll = df[['lat','long']] geometryX = [Point(xy) for xy in zip(pll['long'], pll['lat'])] - + gjlist= [] lbuf = 0.01 xbuf = lbuf#/2.0 - mn = 0 totalm2 = 0.0 for pp in geometryX: - listarray = [] - #listarray.append([pp.x, pp.y]) - listarray.append([pp.x+xbuf, pp.y-lbuf]) - listarray.append([pp.x-xbuf, pp.y-lbuf]) - listarray.append([pp.x-xbuf, pp.y+lbuf]) - listarray.append([pp.x+xbuf, pp.y+lbuf]) # self intersection? - + listarray = [ + [pp.x + xbuf, pp.y - lbuf], + [pp.x - xbuf, pp.y - lbuf], + [pp.x - xbuf, pp.y + lbuf], + [pp.x + xbuf, pp.y + lbuf], + ] + nparray = np.array(listarray) poly = geometry.Polygon(nparray) - + geom = poly geom_area = ops.transform( partial( @@ -141,7 +140,7 @@ def generate_geojson_from_coords_df(df,kbuf): lat1=geom.bounds[1], lat2=geom.bounds[3])), geom) - + # Print the area in m^2 #print(geom_area.area) totalm2 += geom_area.area @@ -149,9 +148,6 @@ def generate_geojson_from_coords_df(df,kbuf): #print(poly.wkt) g1 = shapely.geometry.mapping(poly) gjlist.append(json.dumps(g1)) - #print(json.dumps(g1,indent=1)) - #print(mn) - mn += 1 #print("Generated geojson estimated to consume ",totalm2/1000000.0," square kilometers per instance") return(gjlist) @@ -165,7 +161,7 @@ def get_month_day_range(date): def construct_search_request_for_month(geojson_geometry,dtmonth): xdates = get_month_day_range(dtmonth) - start_date = xdates[0] + start_date = xdates[0] end_date = xdates[1] # get images that overlap with our AOI geometry_filter = { @@ -173,9 +169,9 @@ def construct_search_request_for_month(geojson_geometry,dtmonth): "field_name": "geometry", "config": json.loads(geojson_geometry) } - - gte = start_date.isoformat() + '.000Z' - lte = end_date.isoformat() + '.000Z' + + gte = start_date.isoformat() + '.000Z' + lte = end_date.isoformat() + '.000Z' # get images acquired within a date range date_range_filter = { "type": "DateRangeFilter", @@ -193,24 +189,21 @@ def construct_search_request_for_month(geojson_geometry,dtmonth): "lte": 0.5 } } - + # combine our geo, date, cloud filters combined_filter = { "type": "AndFilter", "config": [geometry_filter, date_range_filter, cloud_cover_filter] } - + #item_type = "PSScene3Band" item_type = "PSScene4Band" - - # API request object - search_request = { - "interval": "day", - "item_types": [item_type], - "filter": combined_filter - } - #print(search_request) - return(search_request) + + return { + "interval": "day", + "item_types": [item_type], + "filter": combined_filter, + } def construct_search_request(geojson_geometry,date_string,date_string_end): start_date = datetime.datetime.strptime(date_string, '%d-%b-%y') @@ -224,9 +217,9 @@ def construct_search_request(geojson_geometry,date_string,date_string_end): #"config": json.loads(geojson_geometry) "config": geojson_geometry } - - gte = start_date.isoformat() + '.000Z' - lte = end_date.isoformat() + '.000Z' + + gte = start_date.isoformat() + '.000Z' + lte = end_date.isoformat() + '.000Z' # get images acquired within a date range date_range_filter = { "type": "DateRangeFilter", @@ -244,24 +237,21 @@ def construct_search_request(geojson_geometry,date_string,date_string_end): "lte": 0.5 } } - + # combine our geo, date, cloud filters combined_filter = { "type": "AndFilter", "config": [geometry_filter, date_range_filter, cloud_cover_filter] } - + #item_type = "PSScene3Band" item_type = "PSScene4Band" - - # API request object - search_request = { - "interval": "day", - "item_types": [item_type], - "filter": combined_filter - } - #print(search_request) - return(search_request) + + return { + "interval": "day", + "item_types": [item_type], + "filter": combined_filter, + } def execute_search_get_json(search_request): @@ -275,29 +265,29 @@ def execute_search_get_json(search_request): def execute_search_get_item_ids(search_request): - search_result = \ - requests.post( - 'https://api.planet.com/data/v1/quick-search', - auth=HTTPBasicAuth(PLANET_API_KEY, ''), - json=search_request) - - #print(json.dumps(search_result.json(), indent=1)) - - # extract image IDs only - image_ids = [feature['id'] for feature in search_result.json()['features']] - print(image_ids) - print(len(image_ids)) - - #item_type = "PSScene3Band" - item_type = "PSScene4Band" - item_ids = [] - - for i in range(len(image_ids)): - id0 = image_ids[i] - id0_url = 'https://api.planet.com/data/v1/item-types/{}/items/{}/assets'.format(item_type, id0) - item_ids.append(id0) - - return(item_ids) + search_result = \ + requests.post( + 'https://api.planet.com/data/v1/quick-search', + auth=HTTPBasicAuth(PLANET_API_KEY, ''), + json=search_request) + + #print(json.dumps(search_result.json(), indent=1)) + + # extract image IDs only + image_ids = [feature['id'] for feature in search_result.json()['features']] + print(image_ids) + print(len(image_ids)) + + item_type = "PSScene4Band" + item_ids = [] + + for image_id in image_ids: + id0 = image_id + id0_url = f'https://api.planet.com/data/v1/item-types/{item_type}/items/{id0}/assets' + + item_ids.append(id0) + + return(item_ids) def get_thumbs_from_search_json(search_json): filenames = [] @@ -305,7 +295,7 @@ def get_thumbs_from_search_json(search_json): image_ids = [feature['id'] for feature in search_json['features']] for i in range(len(tb)): tb_url = search_json['features'][i]["_links"]['thumbnail'] - filestring = str(image_ids[i])+".jpg" + filestring = f"{str(image_ids[i])}.jpg" if os.path.exists(os.path.dirname(filestring)) != True: r = requests.get(tb_url, auth=HTTPBasicAuth(PLANET_API_KEY, ''),allow_redirects=True) filenames.append(filestring) @@ -314,41 +304,31 @@ def get_thumbs_from_search_json(search_json): return(filenames) def get_item_ids_from_search_json(search_json): - image_ids = [feature['id'] for feature in search_json['features']] - #item_type = "PSScene3Band" - item_type = "PSScene4Band" - item_ids = [] - for i in range(len(image_ids)): - id0 = image_ids[i] - id0_url = 'https://api.planet.com/data/v1/item-types/{}/items/{}/assets'.format(item_type, id0) - item_ids.append(id0) - return(item_ids) + image_ids = [feature['id'] for feature in search_json['features']] + item_type = "PSScene4Band" + item_ids = [] + for image_id in image_ids: + id0 = image_id + id0_url = f'https://api.planet.com/data/v1/item-types/{item_type}/items/{id0}/assets' + + item_ids.append(id0) + return(item_ids) #The Data API does not pre-generate assets, so they are not always immediately availiable to download. #In order to download an asset, we first have to activate it. def construct_requestv2(name,aoi,item_ids): - req = { - "name": name, - "products": [ - { - "item_ids": item_ids, - "item_type": "PSScene4Band", - "product_bundle": "analytic" - } - ], - "tools": [ - { - "clip": { - "aoi": { - "type": "Polygon", - "coordinates": aoi + return { + "name": name, + "products": [ + { + "item_ids": item_ids, + "item_type": "PSScene4Band", + "product_bundle": "analytic", } - } - } - ] + ], + "tools": [{"clip": {"aoi": {"type": "Polygon", "coordinates": aoi}}}], } - return(req) def requestv2(req): #This is where we pass the original request: @@ -366,12 +346,7 @@ def get_requestv2_results(): #### This is where we get the results from the original request: ### this will always try to pull the most recent order hp = 'https://api.planet.com/compute/ops/orders/v2' - result = \ - requests.get( - hp, - auth=HTTPBasicAuth(PLANET_API_KEY, '') - ) - return(result) + return requests.get(hp, auth=HTTPBasicAuth(PLANET_API_KEY, '')) # Will return a list of loc_ids, iterate over each using get_download_url below: @@ -381,11 +356,11 @@ def extract_location_ids_v1(result): return(loc_ids) def extract_location_ids(r2js,name): - loc_ids = [] - for order in r2js['orders']: - if(order['name']==name): - loc_ids.append(order["_links"]["_self"]) - return(loc_ids) + return [ + order["_links"]["_self"] + for order in r2js['orders'] + if (order['name'] == name) + ] # We now perform this tasdk in perform_download def get_download_url(loc_ids_self): @@ -398,9 +373,7 @@ def get_download_url(loc_ids_self): #print(result) #print(json.dumps(result.json(), indent=1)) dres = result.json()['_links'] - #dres = spec_result['_links'] - act_ids = [order['location'] for order in dres['results']] - return(act_ids) + return [order['location'] for order in dres['results']] #print(act_ids) #names = [order['name'] for order in dres['results']] #print(names) @@ -461,8 +434,7 @@ def lldist(lat1,lon1,lat2,lon2): epsg_for_utm = 3112 # Using: https://epsg.io/ & http://www.dmap.co.uk/utmworld.htm & https://spatialreference.org/ref/epsg/wgs-84-utm-zone-56s/ #gdf.to_crs(epsg=epsg_for_utm,inplace=True) gdf.to_crs(epsg=3112,inplace=True) - l=gdf.distance(gdf.shift()) - return(l) + return gdf.distance(gdf.shift()) def estimate_quota_usage(search_json): image_ids = [feature['id'] for feature in search_json['features']] @@ -561,19 +533,19 @@ def check_results_for_completion(rjs): if __name__ == "__main__": print("Launching Order_API") - + print("Loading Planet API key from key.txt") with open('key.txt', 'r') as file: PLANET_API_KEY = file.read().replace('\n', '') locfile = sys.argv[1] - if locfile == "resume": - resume_and_sync() - exit() if locfile == "quota": get_remaining_quota() exit() + elif locfile == "resume": + resume_and_sync() + exit() observation_date = sys.argv[2] # 'DD-Mon-YY' observation_date_end = sys.argv[3] # 'DD-Mon-YY' name = sys.argv[4] @@ -583,24 +555,23 @@ def check_results_for_completion(rjs): print("from locations in",locfile) print("Search request is named:",name) df = pd.read_csv(locfile) - geojson_list = generate_geojson_from_coords_df(df,0.05) + geojson_list = generate_geojson_from_coords_df(df,0.05) exsqkm = calculate_sqkm(df,0.05) - + dfwdd_rows = [] - gtodcounter = 0 - for g in geojson_list: + tflist = [] + for gtodcounter, g in enumerate(geojson_list): odfr = df.iloc[gtodcounter].to_dict() - gtodcounter += 1 gj = json.loads(g) print(gj) gjc = gj['coordinates'] search_request = construct_search_request(gj,observation_date,observation_date_end) search_json = execute_search_get_json(search_request) item_ids = get_item_ids_from_search_json(search_json) - print(item_ids) + print(item_ids) gquota = get_remaining_quota() print("Expected to consume ",exsqkm*len(item_ids)," square kilometers of ",gquota," square kilometers remaining") - + gj = json.loads(g) gjc = gj['coordinates'] order_request = construct_requestv2(name,gjc,item_ids) @@ -614,7 +585,7 @@ def check_results_for_completion(rjs): print(msg) regx = re.compile(r'4Band/.+') itrem = regx.search(msg) - trm = re.sub('4Band/','',itrem.group(0)) + trm = re.sub('4Band/', '', itrem[0]) if trm in item_ids: item_ids.remove(trm) order_request = construct_requestv2(name,gjc,item_ids) print(order_request) @@ -630,26 +601,25 @@ def check_results_for_completion(rjs): print(f"Seconds elapsed waiting for activation of requested imagery: {time.time()-waittimestart}") requestv2_json = get_requestv2_results() complete_flag = check_results_for_completion(requestv2_json.json()) - + loc_ids = extract_location_ids(requestv2_json.json(),name) r = re.compile('MS_clip.tif') for i in range(len(loc_ids)): - dfnames = perform_download(name,loc_ids[i]) - tflist = [] - get_remaining_quota() - for fn in dfnames: - if r.search(fn): - print("Acquired image: ",fn) - dfr = odfr.copy() - dfr["filename"] = fn - dfwdd_rows.append(dfr) - #fb = load_image4(fn) - #print(len(fb)) - #plot_bands4(fb,title=fn) - #t = calc_nvdi(fb) - #plt.set_cmap('jet') - #plt.imshow(t) - #plt.show() - + dfnames = perform_download(name,loc_ids[i]) + get_remaining_quota() + for fn in dfnames: + if r.search(fn): + print("Acquired image: ",fn) + dfr = odfr.copy() + dfr["filename"] = fn + dfwdd_rows.append(dfr) + #fb = load_image4(fn) + #print(len(fb)) + #plot_bands4(fb,title=fn) + #t = calc_nvdi(fb) + #plt.set_cmap('jet') + #plt.imshow(t) + #plt.show() + finaldf = pd.DataFrame(dfwdd_rows) finaldf.to_csv(f"{name}_storage_paths.csv")