diff options
Diffstat (limited to 'docs')
| -rw-r--r-- | docs/apidoc/SimpleProducer.rst | 14 | ||||
| -rw-r--r-- | docs/apidoc/kafka.consumer.rst | 46 | ||||
| -rw-r--r-- | docs/apidoc/kafka.coordinator.assignors.rst | 30 | ||||
| -rw-r--r-- | docs/apidoc/kafka.coordinator.rst | 45 | ||||
| -rw-r--r-- | docs/apidoc/kafka.partitioner.rst | 38 | ||||
| -rw-r--r-- | docs/apidoc/kafka.producer.rst | 38 | ||||
| -rw-r--r-- | docs/apidoc/kafka.protocol.rst | 126 | ||||
| -rw-r--r-- | docs/apidoc/kafka.rst | 89 | ||||
| -rw-r--r-- | docs/index.rst | 8 | ||||
| -rw-r--r-- | docs/simple.rst | 162 | ||||
| -rw-r--r-- | docs/tests.rst | 2 |
11 files changed, 0 insertions, 598 deletions
diff --git a/docs/apidoc/SimpleProducer.rst b/docs/apidoc/SimpleProducer.rst deleted file mode 100644 index a509858..0000000 --- a/docs/apidoc/SimpleProducer.rst +++ /dev/null @@ -1,14 +0,0 @@ -SimpleProducer -============== - -.. autoclass:: kafka.producer.SimpleProducer - :members: - :show-inheritance: - -.. autoclass:: kafka.producer.KeyedProducer - :members: - :show-inheritance: - -.. automodule:: kafka.producer.base - :members: - :show-inheritance: diff --git a/docs/apidoc/kafka.consumer.rst b/docs/apidoc/kafka.consumer.rst deleted file mode 100644 index 8595f99..0000000 --- a/docs/apidoc/kafka.consumer.rst +++ /dev/null @@ -1,46 +0,0 @@ -kafka.consumer package -====================== - -Submodules ----------- - -kafka.consumer.base module --------------------------- - -.. automodule:: kafka.consumer.base - :members: - :undoc-members: - :show-inheritance: - -kafka.consumer.kafka module ---------------------------- - -.. automodule:: kafka.consumer.kafka - :members: - :undoc-members: - :show-inheritance: - -kafka.consumer.multiprocess module ----------------------------------- - -.. automodule:: kafka.consumer.multiprocess - :members: - :undoc-members: - :show-inheritance: - -kafka.consumer.simple module ----------------------------- - -.. automodule:: kafka.consumer.simple - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: kafka.consumer - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/apidoc/kafka.coordinator.assignors.rst b/docs/apidoc/kafka.coordinator.assignors.rst deleted file mode 100644 index 87b9f84..0000000 --- a/docs/apidoc/kafka.coordinator.assignors.rst +++ /dev/null @@ -1,30 +0,0 @@ -kafka.coordinator.assignors package -=================================== - -Submodules ----------- - -kafka.coordinator.assignors.abstract module -------------------------------------------- - -.. automodule:: kafka.coordinator.assignors.abstract - :members: - :undoc-members: - :show-inheritance: - -kafka.coordinator.assignors.roundrobin module ---------------------------------------------- - -.. automodule:: kafka.coordinator.assignors.roundrobin - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: kafka.coordinator.assignors - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/apidoc/kafka.coordinator.rst b/docs/apidoc/kafka.coordinator.rst deleted file mode 100644 index e15f638..0000000 --- a/docs/apidoc/kafka.coordinator.rst +++ /dev/null @@ -1,45 +0,0 @@ -kafka.coordinator package -========================= - -Subpackages ------------ - -.. toctree:: - - kafka.coordinator.assignors - -Submodules ----------- - -kafka.coordinator.base module ------------------------------ - -.. automodule:: kafka.coordinator.base - :members: - :undoc-members: - :show-inheritance: - -kafka.coordinator.consumer module ---------------------------------- - -.. automodule:: kafka.coordinator.consumer - :members: - :undoc-members: - :show-inheritance: - -kafka.coordinator.heartbeat module ----------------------------------- - -.. automodule:: kafka.coordinator.heartbeat - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: kafka.coordinator - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/apidoc/kafka.partitioner.rst b/docs/apidoc/kafka.partitioner.rst deleted file mode 100644 index ea215f1..0000000 --- a/docs/apidoc/kafka.partitioner.rst +++ /dev/null @@ -1,38 +0,0 @@ -kafka.partitioner package -========================= - -Submodules ----------- - -kafka.partitioner.base module ------------------------------ - -.. automodule:: kafka.partitioner.base - :members: - :undoc-members: - :show-inheritance: - -kafka.partitioner.hashed module -------------------------------- - -.. automodule:: kafka.partitioner.hashed - :members: - :undoc-members: - :show-inheritance: - -kafka.partitioner.roundrobin module ------------------------------------ - -.. automodule:: kafka.partitioner.roundrobin - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: kafka.partitioner - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/apidoc/kafka.producer.rst b/docs/apidoc/kafka.producer.rst deleted file mode 100644 index bd850bb..0000000 --- a/docs/apidoc/kafka.producer.rst +++ /dev/null @@ -1,38 +0,0 @@ -kafka.producer package -====================== - -Submodules ----------- - -kafka.producer.base module --------------------------- - -.. automodule:: kafka.producer.base - :members: - :undoc-members: - :show-inheritance: - -kafka.producer.keyed module ---------------------------- - -.. automodule:: kafka.producer.keyed - :members: - :undoc-members: - :show-inheritance: - -kafka.producer.simple module ----------------------------- - -.. automodule:: kafka.producer.simple - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: kafka.producer - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/apidoc/kafka.protocol.rst b/docs/apidoc/kafka.protocol.rst deleted file mode 100644 index 4e69aaf..0000000 --- a/docs/apidoc/kafka.protocol.rst +++ /dev/null @@ -1,126 +0,0 @@ -kafka.protocol package -====================== - -Submodules ----------- - -kafka.protocol.abstract module ------------------------------- - -.. automodule:: kafka.protocol.abstract - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol.admin module ---------------------------- - -.. automodule:: kafka.protocol.admin - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol.api module -------------------------- - -.. automodule:: kafka.protocol.api - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol.commit module ----------------------------- - -.. automodule:: kafka.protocol.commit - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol.fetch module ---------------------------- - -.. automodule:: kafka.protocol.fetch - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol.group module ---------------------------- - -.. automodule:: kafka.protocol.group - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol.legacy module ----------------------------- - -.. automodule:: kafka.protocol.legacy - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol.message module ------------------------------ - -.. automodule:: kafka.protocol.message - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol.metadata module ------------------------------- - -.. automodule:: kafka.protocol.metadata - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol.offset module ----------------------------- - -.. automodule:: kafka.protocol.offset - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol.pickle module ----------------------------- - -.. automodule:: kafka.protocol.pickle - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol.produce module ------------------------------ - -.. automodule:: kafka.protocol.produce - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol.struct module ----------------------------- - -.. automodule:: kafka.protocol.struct - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol.types module ---------------------------- - -.. automodule:: kafka.protocol.types - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: kafka.protocol - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/apidoc/kafka.rst b/docs/apidoc/kafka.rst deleted file mode 100644 index a29e063..0000000 --- a/docs/apidoc/kafka.rst +++ /dev/null @@ -1,89 +0,0 @@ -kafka package -============= - -Subpackages ------------ - -.. toctree:: - - kafka.cluster - kafka.consumer - kafka.partitioner - kafka.producer - -Submodules ----------- - -kafka.cluster module --------------------- - -.. automodule:: kafka.cluster - :members: - :undoc-members: - :show-inheritance: - - -kafka.client module -------------------- - -.. automodule:: kafka.client - :members: - :undoc-members: - :show-inheritance: - -kafka.codec module ------------------- - -.. automodule:: kafka.codec - :members: - :undoc-members: - :show-inheritance: - -kafka.common module -------------------- - -.. automodule:: kafka.common - :members: - :undoc-members: - :show-inheritance: - -kafka.conn module ------------------ - -.. automodule:: kafka.conn - :members: - :undoc-members: - :show-inheritance: - -kafka.context module --------------------- - -.. automodule:: kafka.context - :members: - :undoc-members: - :show-inheritance: - -kafka.protocol module ---------------------- - -.. automodule:: kafka.protocol - :members: - :undoc-members: - :show-inheritance: - -kafka.util module ------------------ - -.. automodule:: kafka.util - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: kafka - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/index.rst b/docs/index.rst index 6fa9a0c..2322471 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -139,20 +139,12 @@ method that probes a kafka broker and attempts to identify which version it is running (0.8.0 to 2.3+). -Low-level -********* - -Legacy support is maintained for low-level consumer and producer classes, -SimpleConsumer and SimpleProducer. - - .. toctree:: :hidden: :maxdepth: 2 Usage Overview <usage> API </apidoc/modules> - Simple Clients [deprecated] <simple> install tests compatibility diff --git a/docs/simple.rst b/docs/simple.rst deleted file mode 100644 index afdb975..0000000 --- a/docs/simple.rst +++ /dev/null @@ -1,162 +0,0 @@ -Simple APIs (DEPRECATED) -************************ - - -SimpleConsumer (DEPRECATED) -=========================== - -.. code:: python - - from kafka import SimpleProducer, SimpleClient - - # To consume messages - client = SimpleClient('localhost:9092') - consumer = SimpleConsumer(client, "my-group", "my-topic") - for message in consumer: - # message is raw byte string -- decode if necessary! - # e.g., for unicode: `message.decode('utf-8')` - print(message) - - - # Use multiprocessing for parallel consumers - from kafka import MultiProcessConsumer - - # This will split the number of partitions among two processes - consumer = MultiProcessConsumer(client, "my-group", "my-topic", num_procs=2) - - # This will spawn processes such that each handles 2 partitions max - consumer = MultiProcessConsumer(client, "my-group", "my-topic", - partitions_per_proc=2) - - for message in consumer: - print(message) - - for message in consumer.get_messages(count=5, block=True, timeout=4): - print(message) - - client.close() - - -SimpleProducer (DEPRECATED) -=========================== - -Asynchronous Mode ------------------ - -.. code:: python - - from kafka import SimpleProducer, SimpleClient - - # To send messages asynchronously - client = SimpleClient('localhost:9092') - producer = SimpleProducer(client, async_send=True) - producer.send_messages('my-topic', b'async message') - - # To send messages in batch. You can use any of the available - # producers for doing this. The following producer will collect - # messages in batch and send them to Kafka after 20 messages are - # collected or every 60 seconds - # Notes: - # * If the producer dies before the messages are sent, there will be losses - # * Call producer.stop() to send the messages and cleanup - producer = SimpleProducer(client, - async_send=True, - batch_send_every_n=20, - batch_send_every_t=60) - -Synchronous Mode ----------------- - -.. code:: python - - from kafka import SimpleProducer, SimpleClient - - # To send messages synchronously - client = SimpleClient('localhost:9092') - producer = SimpleProducer(client, async_send=False) - - # Note that the application is responsible for encoding messages to type bytes - producer.send_messages('my-topic', b'some message') - producer.send_messages('my-topic', b'this method', b'is variadic') - - # Send unicode message - producer.send_messages('my-topic', u'你怎么样?'.encode('utf-8')) - - # To wait for acknowledgements - # ACK_AFTER_LOCAL_WRITE : server will wait till the data is written to - # a local log before sending response - # ACK_AFTER_CLUSTER_COMMIT : server will block until the message is committed - # by all in sync replicas before sending a response - producer = SimpleProducer(client, - async_send=False, - req_acks=SimpleProducer.ACK_AFTER_LOCAL_WRITE, - ack_timeout=2000, - sync_fail_on_error=False) - - responses = producer.send_messages('my-topic', b'another message') - for r in responses: - logging.info(r.offset) - - -KeyedProducer (DEPRECATED) -========================== - -.. code:: python - - from kafka import ( - SimpleClient, KeyedProducer, - Murmur2Partitioner, RoundRobinPartitioner) - - kafka = SimpleClient('localhost:9092') - - # HashedPartitioner is default (currently uses python hash()) - producer = KeyedProducer(kafka) - producer.send_messages(b'my-topic', b'key1', b'some message') - producer.send_messages(b'my-topic', b'key2', b'this methode') - - # Murmur2Partitioner attempts to mirror the java client hashing - producer = KeyedProducer(kafka, partitioner=Murmur2Partitioner) - - # Or just produce round-robin (or just use SimpleProducer) - producer = KeyedProducer(kafka, partitioner=RoundRobinPartitioner) - - -SimpleClient (DEPRECATED) -========================= - - -.. code:: python - - import time - from kafka import SimpleClient - from kafka.errors import LeaderNotAvailableError, NotLeaderForPartitionError - from kafka.protocol import create_message - from kafka.structs import ProduceRequestPayload - - kafka = SimpleClient('localhost:9092') - payload = ProduceRequestPayload(topic='my-topic', partition=0, - messages=[create_message("some message")]) - - retries = 5 - resps = [] - while retries and not resps: - retries -= 1 - try: - resps = kafka.send_produce_request( - payloads=[payload], fail_on_error=True) - except LeaderNotAvailableError, NotLeaderForPartitionError: - kafka.load_metadata_for_topics() - time.sleep(1) - - # Other exceptions you might consider handling: - # UnknownTopicOrPartitionError, TopicAuthorizationFailedError, - # RequestTimedOutError, MessageSizeTooLargeError, InvalidTopicError, - # RecordListTooLargeError, InvalidRequiredAcksError, - # NotEnoughReplicasError, NotEnoughReplicasAfterAppendError - - kafka.close() - - resps[0].topic # 'my-topic' - resps[0].partition # 0 - resps[0].error # 0 - resps[0].offset # offset of the first message sent in this request diff --git a/docs/tests.rst b/docs/tests.rst index 5983475..561179c 100644 --- a/docs/tests.rst +++ b/docs/tests.rst @@ -7,8 +7,6 @@ Tests :target: https://travis-ci.org/dpkp/kafka-python Test environments are managed via tox. The test suite is run via pytest. -Individual tests are written using unittest, pytest, and in some cases, -doctest. Linting is run via pylint, but is generally skipped on pypy due to pylint compatibility / performance issues. |