106 aws s3

Accessing files from AWS S3

Uncomment the following line to install leafmap if needed.

In [1]:

# %pip install -U leafmap

# %pip install -U leafmap

In [2]:

from leafmap.common import s3_list_directories, s3_list_files, s3_get_file

from leafmap.common import s3_list_directories, s3_list_files, s3_get_file

In [3]:

bucket = "overturemaps-us-west-2"
prefix = "release"
region = "us-west-2"

bucket = "overturemaps-us-west-2" prefix = "release" region = "us-west-2"

In [4]:

directories = s3_list_directories(bucket, prefix, region=region, return_full_path=False)
directories

directories = s3_list_directories(bucket, prefix, region=region, return_full_path=False) directories

Out[4]:

['2023-04-02-alpha',
 '2023-07-26-alpha.0',
 '2023-10-19-alpha.0',
 '2023-11-14-alpha.0',
 '2023-12-14-alpha.0',
 '2024-01-17-alpha.0',
 '2024-02-15-alpha.0',
 '2024-03-12-alpha.0',
 '2024-04-16-beta.0',
 '2024-05-16-beta.0',
 '2024-06-13-beta.0',
 '2024-06-13-beta.1',
 '2024-07-22.0',
 '2024-08-20.0',
 '2024-09-18.0',
 '2024-10-23.0',
 '2024-11-13.0',
 '2024-12-18.0',
 '2025-01-22.0',
 '2025-02-19.0',
 '2025-03-19.0',
 '2025-03-19.1',
 '2025-04-23.0',
 '2025-05-21.0',
 '2025-06-25.0',
 '2025-07-23.0',
 '2025-08-20.0',
 '2025-08-20.1']

In [5]:

prefix = "release/2025-07-23.0/theme=buildings/type=building"

prefix = "release/2025-07-23.0/theme=buildings/type=building"

In [6]:

files = s3_list_files(
    bucket, prefix, region=region, skip_signature=True, return_full_path=True
)
files[:5]

files = s3_list_files( bucket, prefix, region=region, skip_signature=True, return_full_path=True ) files[:5]

Out[6]:

['s3://overturemaps-us-west-2/release/2025-07-23.0/theme=buildings/type=building/part-00000-afde5bd5-ea80-4567-968a-54b9be7eeafd-c000.zstd.parquet',
 's3://overturemaps-us-west-2/release/2025-07-23.0/theme=buildings/type=building/part-00001-afde5bd5-ea80-4567-968a-54b9be7eeafd-c000.zstd.parquet',
 's3://overturemaps-us-west-2/release/2025-07-23.0/theme=buildings/type=building/part-00002-afde5bd5-ea80-4567-968a-54b9be7eeafd-c000.zstd.parquet',
 's3://overturemaps-us-west-2/release/2025-07-23.0/theme=buildings/type=building/part-00003-afde5bd5-ea80-4567-968a-54b9be7eeafd-c000.zstd.parquet',
 's3://overturemaps-us-west-2/release/2025-07-23.0/theme=buildings/type=building/part-00004-afde5bd5-ea80-4567-968a-54b9be7eeafd-c000.zstd.parquet']

In [7]:

path = "s3://us-west-2.opendata.source.coop/giswqs/nwi/state_boundary/AK.parquet"
output_file = "AK.parquet"

path = "s3://us-west-2.opendata.source.coop/giswqs/nwi/state_boundary/AK.parquet" output_file = "AK.parquet"

In [8]:

s3_get_file(path, region=region, skip_signature=True, output_file=output_file)

s3_get_file(path, region=region, skip_signature=True, output_file=output_file)

Out[8]:

'AK.parquet'