Skip to content

Commit c606867

Browse files
committed
Add singlepart examples to readme
1 parent f62b59d commit c606867

1 file changed

Lines changed: 27 additions & 15 deletions

File tree

README.rst

Lines changed: 27 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -150,21 +150,33 @@ For the sake of simplicity, the examples below assume you have all the dependenc
150150

151151
.. code-block:: python
152152
153-
>>> import os, boto3
154-
>>> from smart_open import open
155-
>>>
156-
>>> # stream content *into* S3 (write mode) using a custom session
157-
>>> session = boto3.Session(
158-
... aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
159-
... aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
160-
... )
161-
>>> url = 's3://smart-open-py37-benchmark-results/test.txt'
162-
>>> with open(url, 'wb', transport_params={'client': session.client('s3')}) as fout:
163-
... bytes_written = fout.write(b'hello world!')
164-
... print(bytes_written)
165-
12
166-
167-
.. code-block:: python
153+
import os, boto3, botocore
154+
from smart_open import open
155+
156+
# stream content *into* S3 (write mode) using a custom client
157+
# this client is thread-safe ref https://github.com/boto/boto3/blob/1.38.41/docs/source/guide/clients.rst?plain=1#L111
158+
config = botocore.client.Config(
159+
max_pool_connections=64,
160+
tcp_keepalive=True,
161+
retries={"max_attempts": 6, "mode": "adaptive"},
162+
)
163+
client = boto3.Session(
164+
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
165+
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
166+
).client("s3", config=config)
167+
with open('s3://smart-open-py37-benchmark-results/test.txt', 'wb', transport_params={'client': client}) as fout:
168+
bytes_written = fout.write(b'hello world!')
169+
print(bytes_written)
170+
171+
# perform a single-part upload to S3 (saves billable API requests, and allows seek() before upload)
172+
with open('s3://smart-open-py37-benchmark-results/test.txt', 'wb', transport_params={'multipart_upload': False}) as fout:
173+
bytes_written = fout.write(b'hello world!')
174+
print(bytes_written)
175+
# now with tempfile.TemporaryFile instead of the default io.BytesIO (to reduce memory footprint)
176+
import tempfile
177+
with tempfile.TemporaryFile() as tmp, open('s3://smart-open-py37-benchmark-results/test.txt', 'wb', transport_params={'multipart_upload': False, 'writebuffer': tmp}) as fout:
178+
bytes_written = fout.write(b'hello world!')
179+
print(bytes_written)
168180
169181
# stream from HDFS
170182
for line in open('hdfs://user/hadoop/my_file.txt', encoding='utf8'):

0 commit comments

Comments
 (0)