From 20c3feb4d2e2565670a4bc09b63216a0fe10da57 Mon Sep 17 00:00:00 2001 From: philipp Date: Sat, 28 Mar 2026 13:04:10 +0100 Subject: [PATCH] Initial commit through Cline Kanban --- .env.example | 6 + .gitignore | 5 + README.md | 134 ++ docs/minimal-product.md | 198 ++ docs/spec.md | 144 ++ index.mjs | 1 + node_modules/.package-lock.json | 17 + node_modules/kafkajs/CHANGELOG.md | 754 ++++++++ node_modules/kafkajs/CONTRIBUTING.md | 40 + node_modules/kafkajs/LICENSE | 24 + node_modules/kafkajs/README.md | 178 ++ node_modules/kafkajs/index.js | 30 + node_modules/kafkajs/package.json | 84 + node_modules/kafkajs/src/admin/index.js | 1606 +++++++++++++++++ .../src/admin/instrumentationEvents.js | 28 + node_modules/kafkajs/src/broker/index.js | 913 ++++++++++ .../src/broker/saslAuthenticator/awsIam.js | 37 + .../src/broker/saslAuthenticator/index.js | 82 + .../broker/saslAuthenticator/oauthBearer.js | 50 + .../src/broker/saslAuthenticator/plain.js | 28 + .../src/broker/saslAuthenticator/scram.js | 325 ++++ .../src/broker/saslAuthenticator/scram256.js | 10 + .../src/broker/saslAuthenticator/scram512.js | 10 + .../kafkajs/src/cluster/brokerPool.js | 349 ++++ .../src/cluster/connectionPoolBuilder.js | 117 ++ node_modules/kafkajs/src/cluster/index.js | 539 ++++++ node_modules/kafkajs/src/constants.js | 9 + .../kafkajs/src/consumer/assignerProtocol.js | 87 + .../kafkajs/src/consumer/assigners/index.js | 5 + .../assigners/roundRobinAssigner/index.js | 81 + node_modules/kafkajs/src/consumer/batch.js | 112 ++ .../kafkajs/src/consumer/consumerGroup.js | 759 ++++++++ .../kafkajs/src/consumer/fetchManager.js | 99 + node_modules/kafkajs/src/consumer/fetcher.js | 86 + .../src/consumer/filterAbortedMessages.js | 64 + node_modules/kafkajs/src/consumer/index.js | 522 ++++++ .../src/consumer/instrumentationEvents.js | 40 + .../src/consumer/offsetManager/index.js | 384 ++++ .../initializeConsumerOffsets.js | 26 + .../consumer/offsetManager/isInvalidOffset.js | 3 + node_modules/kafkajs/src/consumer/runner.js | 518 ++++++ .../kafkajs/src/consumer/seekOffsets.js | 27 + .../kafkajs/src/consumer/subscriptionState.js | 123 ++ node_modules/kafkajs/src/consumer/worker.js | 40 + .../kafkajs/src/consumer/workerQueue.js | 40 + node_modules/kafkajs/src/env.js | 4 + node_modules/kafkajs/src/errors.js | 309 ++++ node_modules/kafkajs/src/index.js | 212 +++ .../kafkajs/src/instrumentation/emitter.js | 34 + .../kafkajs/src/instrumentation/event.js | 23 + .../kafkajs/src/instrumentation/eventType.js | 2 + node_modules/kafkajs/src/loggers/console.js | 21 + node_modules/kafkajs/src/loggers/index.js | 68 + .../kafkajs/src/network/connection.js | 543 ++++++ .../kafkajs/src/network/connectionPool.js | 65 + .../kafkajs/src/network/connectionStatus.js | 12 + .../src/network/instrumentationEvents.js | 8 + .../kafkajs/src/network/requestQueue/index.js | 323 ++++ .../src/network/requestQueue/socketRequest.js | 168 ++ node_modules/kafkajs/src/network/socket.js | 32 + .../kafkajs/src/network/socketFactory.js | 22 + .../kafkajs/src/producer/createTopicData.js | 11 + .../kafkajs/src/producer/eosManager/index.js | 464 +++++ .../eosManager/transactionStateMachine.js | 79 + .../producer/eosManager/transactionStates.js | 7 + .../src/producer/groupMessagesPerPartition.js | 11 + node_modules/kafkajs/src/producer/index.js | 246 +++ .../src/producer/instrumentationEvents.js | 28 + .../kafkajs/src/producer/messageProducer.js | 132 ++ .../producer/partitioners/default/index.js | 4 + .../producer/partitioners/default/murmur2.js | 53 + .../src/producer/partitioners/index.js | 14 + .../src/producer/partitioners/legacy/index.js | 4 + .../producer/partitioners/legacy/murmur2.js | 51 + .../partitioners/legacy/partitioner.js | 47 + .../partitioners/legacy/randomBytes.js | 31 + .../src/producer/responseSerializer.js | 4 + .../kafkajs/src/producer/sendMessages.js | 170 ++ .../kafkajs/src/protocol/aclOperationTypes.js | 65 + .../src/protocol/aclPermissionTypes.js | 29 + .../kafkajs/src/protocol/aclResourceTypes.js | 42 + .../src/protocol/configResourceTypes.js | 9 + .../kafkajs/src/protocol/configSource.js | 12 + .../kafkajs/src/protocol/coordinatorTypes.js | 12 + node_modules/kafkajs/src/protocol/crc32.js | 270 +++ node_modules/kafkajs/src/protocol/decoder.js | 309 ++++ node_modules/kafkajs/src/protocol/encoder.js | 405 +++++ node_modules/kafkajs/src/protocol/error.js | 601 ++++++ .../kafkajs/src/protocol/isolationLevel.js | 15 + .../src/protocol/message/compression/gzip.js | 23 + .../src/protocol/message/compression/index.js | 38 + .../kafkajs/src/protocol/message/decoder.js | 37 + .../kafkajs/src/protocol/message/index.js | 6 + .../src/protocol/message/v0/decoder.js | 5 + .../kafkajs/src/protocol/message/v0/index.js | 24 + .../src/protocol/message/v1/decoder.js | 6 + .../kafkajs/src/protocol/message/v1/index.js | 26 + .../src/protocol/messageSet/decoder.js | 91 + .../kafkajs/src/protocol/messageSet/index.js | 41 + .../src/protocol/recordBatch/crc32C/crc32C.js | 85 + .../src/protocol/recordBatch/crc32C/index.js | 4 + .../protocol/recordBatch/header/v0/decoder.js | 4 + .../protocol/recordBatch/header/v0/index.js | 12 + .../protocol/recordBatch/record/v0/decoder.js | 69 + .../protocol/recordBatch/record/v0/index.js | 67 + .../src/protocol/recordBatch/v0/decoder.js | 118 ++ .../src/protocol/recordBatch/v0/index.js | 93 + node_modules/kafkajs/src/protocol/request.js | 13 + .../requests/addOffsetsToTxn/index.js | 17 + .../requests/addOffsetsToTxn/v0/request.js | 23 + .../requests/addOffsetsToTxn/v0/response.js | 33 + .../requests/addOffsetsToTxn/v1/request.js | 20 + .../requests/addOffsetsToTxn/v1/response.js | 24 + .../requests/addPartitionsToTxn/index.js | 17 + .../requests/addPartitionsToTxn/v0/request.js | 33 + .../addPartitionsToTxn/v0/response.js | 51 + .../requests/addPartitionsToTxn/v1/request.js | 22 + .../addPartitionsToTxn/v1/response.js | 28 + .../protocol/requests/alterConfigs/index.js | 17 + .../requests/alterConfigs/v0/request.js | 37 + .../requests/alterConfigs/v0/response.js | 44 + .../requests/alterConfigs/v1/request.js | 25 + .../requests/alterConfigs/v1/response.js | 29 + .../alterPartitionReassignments/index.js | 12 + .../alterPartitionReassignments/v0/request.js | 43 + .../v0/response.js | 88 + .../kafkajs/src/protocol/requests/apiKeys.js | 49 + .../protocol/requests/apiVersions/index.js | 24 + .../requests/apiVersions/v0/request.js | 13 + .../requests/apiVersions/v0/response.js | 43 + .../requests/apiVersions/v1/request.js | 5 + .../requests/apiVersions/v1/response.js | 49 + .../requests/apiVersions/v2/request.js | 5 + .../requests/apiVersions/v2/response.js | 29 + .../src/protocol/requests/createAcls/index.js | 17 + .../requests/createAcls/v0/request.js | 39 + .../requests/createAcls/v0/response.js | 43 + .../requests/createAcls/v1/request.js | 42 + .../requests/createAcls/v1/response.js | 27 + .../requests/createPartitions/index.js | 17 + .../requests/createPartitions/v0/request.js | 36 + .../requests/createPartitions/v0/response.js | 42 + .../requests/createPartitions/v1/request.js | 15 + .../requests/createPartitions/v1/response.js | 28 + .../protocol/requests/createTopics/index.js | 27 + .../requests/createTopics/v0/request.js | 49 + .../requests/createTopics/v0/response.js | 43 + .../requests/createTopics/v1/request.js | 53 + .../requests/createTopics/v1/response.js | 30 + .../requests/createTopics/v2/request.js | 20 + .../requests/createTopics/v2/response.js | 32 + .../requests/createTopics/v3/request.js | 20 + .../requests/createTopics/v3/response.js | 28 + .../src/protocol/requests/deleteAcls/index.js | 17 + .../requests/deleteAcls/v0/request.js | 39 + .../requests/deleteAcls/v0/response.js | 75 + .../requests/deleteAcls/v1/request.js | 42 + .../requests/deleteAcls/v1/response.js | 60 + .../protocol/requests/deleteGroups/index.js | 17 + .../requests/deleteGroups/v0/request.js | 22 + .../requests/deleteGroups/v0/response.js | 39 + .../requests/deleteGroups/v1/request.js | 7 + .../requests/deleteGroups/v1/response.js | 27 + .../protocol/requests/deleteRecords/index.js | 17 + .../requests/deleteRecords/v0/request.js | 30 + .../requests/deleteRecords/v0/response.js | 65 + .../requests/deleteRecords/v1/request.js | 13 + .../requests/deleteRecords/v1/response.js | 34 + .../protocol/requests/deleteTopics/index.js | 17 + .../requests/deleteTopics/v0/request.js | 16 + .../requests/deleteTopics/v0/response.js | 37 + .../requests/deleteTopics/v1/request.js | 10 + .../requests/deleteTopics/v1/response.js | 36 + .../protocol/requests/describeAcls/index.js | 39 + .../requests/describeAcls/v0/request.js | 27 + .../requests/describeAcls/v0/response.js | 58 + .../requests/describeAcls/v1/request.js | 37 + .../requests/describeAcls/v1/response.js | 57 + .../requests/describeConfigs/index.js | 22 + .../requests/describeConfigs/v0/request.js | 29 + .../requests/describeConfigs/v0/response.js | 98 + .../requests/describeConfigs/v1/request.js | 31 + .../requests/describeConfigs/v1/response.js | 72 + .../requests/describeConfigs/v2/request.js | 17 + .../requests/describeConfigs/v2/response.js | 39 + .../protocol/requests/describeGroups/index.js | 22 + .../requests/describeGroups/v0/request.js | 19 + .../requests/describeGroups/v0/response.js | 58 + .../requests/describeGroups/v1/request.js | 8 + .../requests/describeGroups/v1/response.js | 52 + .../requests/describeGroups/v2/request.js | 8 + .../requests/describeGroups/v2/response.js | 36 + .../src/protocol/requests/endTxn/index.js | 23 + .../protocol/requests/endTxn/v0/request.js | 23 + .../protocol/requests/endTxn/v0/response.js | 33 + .../protocol/requests/endTxn/v1/request.js | 14 + .../protocol/requests/endTxn/v1/response.js | 25 + .../src/protocol/requests/fetch/index.js | 251 +++ .../src/protocol/requests/fetch/v0/request.js | 57 + .../protocol/requests/fetch/v0/response.js | 65 + .../src/protocol/requests/fetch/v1/request.js | 5 + .../protocol/requests/fetch/v1/response.js | 44 + .../protocol/requests/fetch/v10/request.js | 55 + .../protocol/requests/fetch/v10/response.js | 26 + .../protocol/requests/fetch/v11/request.js | 84 + .../protocol/requests/fetch/v11/response.js | 69 + .../src/protocol/requests/fetch/v2/request.js | 5 + .../protocol/requests/fetch/v2/response.js | 19 + .../src/protocol/requests/fetch/v3/request.js | 62 + .../protocol/requests/fetch/v3/response.js | 19 + .../requests/fetch/v4/decodeMessages.js | 46 + .../src/protocol/requests/fetch/v4/request.js | 51 + .../protocol/requests/fetch/v4/response.js | 55 + .../src/protocol/requests/fetch/v5/request.js | 53 + .../protocol/requests/fetch/v5/response.js | 57 + .../src/protocol/requests/fetch/v6/request.js | 38 + .../protocol/requests/fetch/v6/response.js | 24 + .../src/protocol/requests/fetch/v7/request.js | 73 + .../protocol/requests/fetch/v7/response.js | 63 + .../src/protocol/requests/fetch/v8/request.js | 54 + .../protocol/requests/fetch/v8/response.js | 67 + .../src/protocol/requests/fetch/v9/request.js | 81 + .../protocol/requests/fetch/v9/response.js | 26 + .../requests/findCoordinator/index.js | 24 + .../requests/findCoordinator/v0/request.js | 16 + .../requests/findCoordinator/v0/response.js | 42 + .../requests/findCoordinator/v1/request.js | 17 + .../requests/findCoordinator/v1/response.js | 48 + .../requests/findCoordinator/v2/request.js | 10 + .../requests/findCoordinator/v2/response.js | 30 + .../src/protocol/requests/heartbeat/index.js | 39 + .../protocol/requests/heartbeat/v0/request.js | 21 + .../requests/heartbeat/v0/response.js | 29 + .../protocol/requests/heartbeat/v1/request.js | 11 + .../requests/heartbeat/v1/response.js | 24 + .../protocol/requests/heartbeat/v2/request.js | 11 + .../requests/heartbeat/v2/response.js | 24 + .../protocol/requests/heartbeat/v3/request.js | 26 + .../requests/heartbeat/v3/response.js | 11 + .../kafkajs/src/protocol/requests/index.js | 106 ++ .../protocol/requests/initProducerId/index.js | 17 + .../requests/initProducerId/v0/request.js | 17 + .../requests/initProducerId/v0/response.js | 37 + .../requests/initProducerId/v1/request.js | 10 + .../requests/initProducerId/v1/response.js | 27 + .../src/protocol/requests/joinGroup/index.js | 135 ++ .../protocol/requests/joinGroup/v0/request.js | 31 + .../requests/joinGroup/v0/response.js | 46 + .../protocol/requests/joinGroup/v1/request.js | 40 + .../requests/joinGroup/v1/response.js | 18 + .../protocol/requests/joinGroup/v2/request.js | 33 + .../requests/joinGroup/v2/response.js | 42 + .../protocol/requests/joinGroup/v3/request.js | 33 + .../requests/joinGroup/v3/response.js | 32 + .../protocol/requests/joinGroup/v4/request.js | 36 + .../requests/joinGroup/v4/response.js | 39 + .../protocol/requests/joinGroup/v5/request.js | 46 + .../requests/joinGroup/v5/response.js | 67 + .../src/protocol/requests/leaveGroup/index.js | 39 + .../requests/leaveGroup/v0/request.js | 17 + .../requests/leaveGroup/v0/response.js | 29 + .../requests/leaveGroup/v1/request.js | 10 + .../requests/leaveGroup/v1/response.js | 24 + .../requests/leaveGroup/v2/request.js | 10 + .../requests/leaveGroup/v2/response.js | 24 + .../requests/leaveGroup/v3/request.js | 29 + .../requests/leaveGroup/v3/response.js | 46 + .../src/protocol/requests/listGroups/index.js | 22 + .../requests/listGroups/v0/request.js | 17 + .../requests/listGroups/v0/response.js | 40 + .../requests/listGroups/v1/request.js | 7 + .../requests/listGroups/v1/response.js | 30 + .../requests/listGroups/v2/request.js | 7 + .../requests/listGroups/v2/response.js | 27 + .../protocol/requests/listOffsets/index.js | 32 + .../requests/listOffsets/v0/request.js | 46 + .../requests/listOffsets/v0/response.js | 49 + .../requests/listOffsets/v1/request.js | 28 + .../requests/listOffsets/v1/response.js | 49 + .../requests/listOffsets/v2/request.js | 32 + .../requests/listOffsets/v2/response.js | 51 + .../requests/listOffsets/v3/request.js | 14 + .../requests/listOffsets/v3/response.js | 30 + .../listPartitionReassignments/index.js | 12 + .../listPartitionReassignments/v0/request.js | 34 + .../listPartitionReassignments/v0/response.js | 72 + .../src/protocol/requests/metadata/index.js | 42 + .../protocol/requests/metadata/v0/request.js | 16 + .../protocol/requests/metadata/v0/response.js | 73 + .../protocol/requests/metadata/v1/request.js | 16 + .../protocol/requests/metadata/v1/response.js | 58 + .../protocol/requests/metadata/v2/request.js | 8 + .../protocol/requests/metadata/v2/response.js | 60 + .../protocol/requests/metadata/v3/request.js | 8 + .../protocol/requests/metadata/v3/response.js | 62 + .../protocol/requests/metadata/v4/request.js | 17 + .../protocol/requests/metadata/v4/response.js | 28 + .../protocol/requests/metadata/v5/request.js | 10 + .../protocol/requests/metadata/v5/response.js | 64 + .../protocol/requests/metadata/v6/request.js | 10 + .../protocol/requests/metadata/v6/response.js | 41 + .../protocol/requests/offsetCommit/index.js | 75 + .../requests/offsetCommit/v0/request.js | 33 + .../requests/offsetCommit/v0/response.js | 45 + .../requests/offsetCommit/v1/request.js | 41 + .../requests/offsetCommit/v1/response.js | 15 + .../requests/offsetCommit/v2/request.js | 41 + .../requests/offsetCommit/v2/response.js | 15 + .../requests/offsetCommit/v3/request.js | 20 + .../requests/offsetCommit/v3/response.js | 35 + .../requests/offsetCommit/v4/request.js | 20 + .../requests/offsetCommit/v4/response.js | 29 + .../requests/offsetCommit/v5/request.js | 41 + .../requests/offsetCommit/v5/response.js | 15 + .../protocol/requests/offsetFetch/index.js | 27 + .../requests/offsetFetch/v1/request.js | 28 + .../requests/offsetFetch/v1/response.js | 49 + .../requests/offsetFetch/v2/request.js | 13 + .../requests/offsetFetch/v2/response.js | 55 + .../requests/offsetFetch/v3/request.js | 28 + .../requests/offsetFetch/v3/response.js | 41 + .../requests/offsetFetch/v4/request.js | 13 + .../requests/offsetFetch/v4/response.js | 32 + .../src/protocol/requests/produce/index.js | 102 ++ .../protocol/requests/produce/v0/request.js | 86 + .../protocol/requests/produce/v0/response.js | 47 + .../protocol/requests/produce/v1/request.js | 8 + .../protocol/requests/produce/v1/response.js | 38 + .../protocol/requests/produce/v2/request.js | 66 + .../protocol/requests/produce/v2/response.js | 40 + .../protocol/requests/produce/v3/request.js | 119 ++ .../protocol/requests/produce/v3/response.js | 54 + .../protocol/requests/produce/v4/request.js | 35 + .../protocol/requests/produce/v4/response.js | 18 + .../protocol/requests/produce/v5/request.js | 35 + .../protocol/requests/produce/v5/response.js | 43 + .../protocol/requests/produce/v6/request.js | 38 + .../protocol/requests/produce/v6/response.js | 32 + .../protocol/requests/produce/v7/request.js | 38 + .../protocol/requests/produce/v7/response.js | 19 + .../requests/saslAuthenticate/index.js | 17 + .../requests/saslAuthenticate/v0/request.js | 19 + .../requests/saslAuthenticate/v0/response.js | 59 + .../requests/saslAuthenticate/v1/request.js | 11 + .../requests/saslAuthenticate/v1/response.js | 37 + .../protocol/requests/saslHandshake/index.js | 17 + .../requests/saslHandshake/v0/request.js | 17 + .../requests/saslHandshake/v0/response.js | 33 + .../requests/saslHandshake/v1/request.js | 3 + .../requests/saslHandshake/v1/response.js | 6 + .../src/protocol/requests/syncGroup/index.js | 39 + .../protocol/requests/syncGroup/v0/request.js | 29 + .../requests/syncGroup/v0/response.js | 33 + .../protocol/requests/syncGroup/v1/request.js | 14 + .../requests/syncGroup/v1/response.js | 29 + .../protocol/requests/syncGroup/v2/request.js | 14 + .../requests/syncGroup/v2/response.js | 26 + .../protocol/requests/syncGroup/v3/request.js | 40 + .../requests/syncGroup/v3/response.js | 12 + .../requests/txnOffsetCommit/index.js | 23 + .../requests/txnOffsetCommit/v0/request.js | 41 + .../requests/txnOffsetCommit/v0/response.js | 51 + .../requests/txnOffsetCommit/v1/request.js | 20 + .../requests/txnOffsetCommit/v1/response.js | 29 + .../src/protocol/resourcePatternTypes.js | 46 + .../kafkajs/src/protocol/sasl/awsIam/index.js | 4 + .../src/protocol/sasl/awsIam/request.js | 11 + .../src/protocol/sasl/awsIam/response.js | 4 + .../src/protocol/sasl/oauthBearer/index.js | 4 + .../src/protocol/sasl/oauthBearer/request.js | 62 + .../src/protocol/sasl/oauthBearer/response.js | 4 + .../kafkajs/src/protocol/sasl/plain/index.js | 4 + .../src/protocol/sasl/plain/request.js | 28 + .../src/protocol/sasl/plain/response.js | 4 + .../sasl/scram/finalMessage/request.js | 5 + .../sasl/scram/finalMessage/response.js | 1 + .../sasl/scram/firstMessage/request.js | 22 + .../sasl/scram/firstMessage/response.js | 23 + .../kafkajs/src/protocol/sasl/scram/index.js | 10 + .../kafkajs/src/protocol/timestampTypes.js | 15 + node_modules/kafkajs/src/retry/defaults.js | 7 + .../kafkajs/src/retry/defaults.test.js | 7 + node_modules/kafkajs/src/retry/index.js | 77 + node_modules/kafkajs/src/utils/arrayDiff.js | 14 + node_modules/kafkajs/src/utils/groupBy.js | 10 + node_modules/kafkajs/src/utils/lock.js | 63 + node_modules/kafkajs/src/utils/long.js | 343 ++++ node_modules/kafkajs/src/utils/mapValues.js | 10 + node_modules/kafkajs/src/utils/once.js | 10 + node_modules/kafkajs/src/utils/seq.js | 9 + .../kafkajs/src/utils/sharedPromiseTo.js | 18 + node_modules/kafkajs/src/utils/shuffle.js | 25 + node_modules/kafkajs/src/utils/sleep.js | 4 + node_modules/kafkajs/src/utils/swapObject.js | 3 + node_modules/kafkajs/src/utils/uniq.js | 3 + node_modules/kafkajs/src/utils/waitFor.js | 50 + node_modules/kafkajs/src/utils/websiteUrl.js | 7 + node_modules/kafkajs/types/index.d.ts | 1311 ++++++++++++++ package-lock.json | 24 + package.json | 14 + src/apps/dummy-consumer.mjs | 46 + src/apps/near-intents-ingest.mjs | 40 + src/bus/kafka.mjs | 27 + src/bus/kafka/consumer.mjs | 16 + src/bus/kafka/producer.mjs | 21 + src/core/env.mjs | 13 + src/core/event-envelope.mjs | 14 + src/core/log.mjs | 31 + src/core/pair-filter.mjs | 17 + src/lib/config.mjs | 33 + src/lib/env.mjs | 13 + src/lib/event-envelope.mjs | 37 + src/venues/near-intents/ingest.mjs | 5 + src/venues/near-intents/normalize.mjs | 50 + src/venues/near-intents/ws.mjs | 162 ++ 415 files changed, 26235 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 README.md create mode 100644 docs/minimal-product.md create mode 100644 docs/spec.md create mode 100644 index.mjs create mode 100644 node_modules/.package-lock.json create mode 100644 node_modules/kafkajs/CHANGELOG.md create mode 100644 node_modules/kafkajs/CONTRIBUTING.md create mode 100644 node_modules/kafkajs/LICENSE create mode 100644 node_modules/kafkajs/README.md create mode 100644 node_modules/kafkajs/index.js create mode 100644 node_modules/kafkajs/package.json create mode 100644 node_modules/kafkajs/src/admin/index.js create mode 100644 node_modules/kafkajs/src/admin/instrumentationEvents.js create mode 100644 node_modules/kafkajs/src/broker/index.js create mode 100644 node_modules/kafkajs/src/broker/saslAuthenticator/awsIam.js create mode 100644 node_modules/kafkajs/src/broker/saslAuthenticator/index.js create mode 100644 node_modules/kafkajs/src/broker/saslAuthenticator/oauthBearer.js create mode 100644 node_modules/kafkajs/src/broker/saslAuthenticator/plain.js create mode 100644 node_modules/kafkajs/src/broker/saslAuthenticator/scram.js create mode 100644 node_modules/kafkajs/src/broker/saslAuthenticator/scram256.js create mode 100644 node_modules/kafkajs/src/broker/saslAuthenticator/scram512.js create mode 100644 node_modules/kafkajs/src/cluster/brokerPool.js create mode 100644 node_modules/kafkajs/src/cluster/connectionPoolBuilder.js create mode 100644 node_modules/kafkajs/src/cluster/index.js create mode 100644 node_modules/kafkajs/src/constants.js create mode 100644 node_modules/kafkajs/src/consumer/assignerProtocol.js create mode 100644 node_modules/kafkajs/src/consumer/assigners/index.js create mode 100644 node_modules/kafkajs/src/consumer/assigners/roundRobinAssigner/index.js create mode 100644 node_modules/kafkajs/src/consumer/batch.js create mode 100644 node_modules/kafkajs/src/consumer/consumerGroup.js create mode 100644 node_modules/kafkajs/src/consumer/fetchManager.js create mode 100644 node_modules/kafkajs/src/consumer/fetcher.js create mode 100644 node_modules/kafkajs/src/consumer/filterAbortedMessages.js create mode 100644 node_modules/kafkajs/src/consumer/index.js create mode 100644 node_modules/kafkajs/src/consumer/instrumentationEvents.js create mode 100644 node_modules/kafkajs/src/consumer/offsetManager/index.js create mode 100644 node_modules/kafkajs/src/consumer/offsetManager/initializeConsumerOffsets.js create mode 100644 node_modules/kafkajs/src/consumer/offsetManager/isInvalidOffset.js create mode 100644 node_modules/kafkajs/src/consumer/runner.js create mode 100644 node_modules/kafkajs/src/consumer/seekOffsets.js create mode 100644 node_modules/kafkajs/src/consumer/subscriptionState.js create mode 100644 node_modules/kafkajs/src/consumer/worker.js create mode 100644 node_modules/kafkajs/src/consumer/workerQueue.js create mode 100644 node_modules/kafkajs/src/env.js create mode 100644 node_modules/kafkajs/src/errors.js create mode 100644 node_modules/kafkajs/src/index.js create mode 100644 node_modules/kafkajs/src/instrumentation/emitter.js create mode 100644 node_modules/kafkajs/src/instrumentation/event.js create mode 100644 node_modules/kafkajs/src/instrumentation/eventType.js create mode 100644 node_modules/kafkajs/src/loggers/console.js create mode 100644 node_modules/kafkajs/src/loggers/index.js create mode 100644 node_modules/kafkajs/src/network/connection.js create mode 100644 node_modules/kafkajs/src/network/connectionPool.js create mode 100644 node_modules/kafkajs/src/network/connectionStatus.js create mode 100644 node_modules/kafkajs/src/network/instrumentationEvents.js create mode 100644 node_modules/kafkajs/src/network/requestQueue/index.js create mode 100644 node_modules/kafkajs/src/network/requestQueue/socketRequest.js create mode 100644 node_modules/kafkajs/src/network/socket.js create mode 100644 node_modules/kafkajs/src/network/socketFactory.js create mode 100644 node_modules/kafkajs/src/producer/createTopicData.js create mode 100644 node_modules/kafkajs/src/producer/eosManager/index.js create mode 100644 node_modules/kafkajs/src/producer/eosManager/transactionStateMachine.js create mode 100644 node_modules/kafkajs/src/producer/eosManager/transactionStates.js create mode 100644 node_modules/kafkajs/src/producer/groupMessagesPerPartition.js create mode 100644 node_modules/kafkajs/src/producer/index.js create mode 100644 node_modules/kafkajs/src/producer/instrumentationEvents.js create mode 100644 node_modules/kafkajs/src/producer/messageProducer.js create mode 100644 node_modules/kafkajs/src/producer/partitioners/default/index.js create mode 100644 node_modules/kafkajs/src/producer/partitioners/default/murmur2.js create mode 100644 node_modules/kafkajs/src/producer/partitioners/index.js create mode 100644 node_modules/kafkajs/src/producer/partitioners/legacy/index.js create mode 100644 node_modules/kafkajs/src/producer/partitioners/legacy/murmur2.js create mode 100644 node_modules/kafkajs/src/producer/partitioners/legacy/partitioner.js create mode 100644 node_modules/kafkajs/src/producer/partitioners/legacy/randomBytes.js create mode 100644 node_modules/kafkajs/src/producer/responseSerializer.js create mode 100644 node_modules/kafkajs/src/producer/sendMessages.js create mode 100644 node_modules/kafkajs/src/protocol/aclOperationTypes.js create mode 100644 node_modules/kafkajs/src/protocol/aclPermissionTypes.js create mode 100644 node_modules/kafkajs/src/protocol/aclResourceTypes.js create mode 100644 node_modules/kafkajs/src/protocol/configResourceTypes.js create mode 100644 node_modules/kafkajs/src/protocol/configSource.js create mode 100644 node_modules/kafkajs/src/protocol/coordinatorTypes.js create mode 100644 node_modules/kafkajs/src/protocol/crc32.js create mode 100644 node_modules/kafkajs/src/protocol/decoder.js create mode 100644 node_modules/kafkajs/src/protocol/encoder.js create mode 100644 node_modules/kafkajs/src/protocol/error.js create mode 100644 node_modules/kafkajs/src/protocol/isolationLevel.js create mode 100644 node_modules/kafkajs/src/protocol/message/compression/gzip.js create mode 100644 node_modules/kafkajs/src/protocol/message/compression/index.js create mode 100644 node_modules/kafkajs/src/protocol/message/decoder.js create mode 100644 node_modules/kafkajs/src/protocol/message/index.js create mode 100644 node_modules/kafkajs/src/protocol/message/v0/decoder.js create mode 100644 node_modules/kafkajs/src/protocol/message/v0/index.js create mode 100644 node_modules/kafkajs/src/protocol/message/v1/decoder.js create mode 100644 node_modules/kafkajs/src/protocol/message/v1/index.js create mode 100644 node_modules/kafkajs/src/protocol/messageSet/decoder.js create mode 100644 node_modules/kafkajs/src/protocol/messageSet/index.js create mode 100644 node_modules/kafkajs/src/protocol/recordBatch/crc32C/crc32C.js create mode 100644 node_modules/kafkajs/src/protocol/recordBatch/crc32C/index.js create mode 100644 node_modules/kafkajs/src/protocol/recordBatch/header/v0/decoder.js create mode 100644 node_modules/kafkajs/src/protocol/recordBatch/header/v0/index.js create mode 100644 node_modules/kafkajs/src/protocol/recordBatch/record/v0/decoder.js create mode 100644 node_modules/kafkajs/src/protocol/recordBatch/record/v0/index.js create mode 100644 node_modules/kafkajs/src/protocol/recordBatch/v0/decoder.js create mode 100644 node_modules/kafkajs/src/protocol/recordBatch/v0/index.js create mode 100644 node_modules/kafkajs/src/protocol/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/alterConfigs/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/alterConfigs/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/alterConfigs/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/alterConfigs/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/alterConfigs/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/alterPartitionReassignments/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/alterPartitionReassignments/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/alterPartitionReassignments/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/apiKeys.js create mode 100644 node_modules/kafkajs/src/protocol/requests/apiVersions/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/apiVersions/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/apiVersions/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/apiVersions/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/apiVersions/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/apiVersions/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/apiVersions/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createAcls/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createAcls/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createAcls/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createAcls/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createAcls/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createPartitions/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createPartitions/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createPartitions/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createPartitions/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createPartitions/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createTopics/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createTopics/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createTopics/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createTopics/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createTopics/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createTopics/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createTopics/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createTopics/v3/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/createTopics/v3/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteAcls/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteAcls/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteAcls/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteAcls/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteAcls/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteGroups/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteGroups/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteGroups/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteGroups/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteGroups/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteRecords/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteRecords/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteRecords/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteRecords/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteRecords/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteTopics/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteTopics/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteTopics/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteTopics/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/deleteTopics/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeAcls/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeAcls/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeAcls/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeAcls/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeAcls/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeConfigs/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeConfigs/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeConfigs/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeConfigs/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeConfigs/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeConfigs/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeConfigs/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeGroups/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeGroups/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeGroups/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeGroups/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeGroups/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeGroups/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/describeGroups/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/endTxn/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/endTxn/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/endTxn/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/endTxn/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/endTxn/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v10/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v10/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v11/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v11/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v3/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v3/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v4/decodeMessages.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v4/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v4/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v5/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v5/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v6/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v6/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v7/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v7/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v8/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v8/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v9/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/fetch/v9/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/findCoordinator/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/findCoordinator/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/findCoordinator/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/findCoordinator/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/findCoordinator/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/findCoordinator/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/findCoordinator/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/heartbeat/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/heartbeat/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/heartbeat/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/heartbeat/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/heartbeat/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/heartbeat/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/heartbeat/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/heartbeat/v3/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/heartbeat/v3/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/initProducerId/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/initProducerId/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/initProducerId/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/initProducerId/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/initProducerId/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/joinGroup/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/joinGroup/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/joinGroup/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/joinGroup/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/joinGroup/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/joinGroup/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/joinGroup/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/joinGroup/v3/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/joinGroup/v3/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/joinGroup/v4/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/joinGroup/v4/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/joinGroup/v5/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/joinGroup/v5/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/leaveGroup/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/leaveGroup/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/leaveGroup/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/leaveGroup/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/leaveGroup/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/leaveGroup/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/leaveGroup/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/leaveGroup/v3/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/leaveGroup/v3/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listGroups/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listGroups/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listGroups/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listGroups/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listGroups/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listGroups/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listGroups/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listOffsets/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listOffsets/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listOffsets/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listOffsets/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listOffsets/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listOffsets/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listOffsets/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listOffsets/v3/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listOffsets/v3/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listPartitionReassignments/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listPartitionReassignments/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/listPartitionReassignments/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v3/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v3/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v4/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v4/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v5/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v5/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v6/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/metadata/v6/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetCommit/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetCommit/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetCommit/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetCommit/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetCommit/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetCommit/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetCommit/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetCommit/v3/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetCommit/v3/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetCommit/v4/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetCommit/v4/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetCommit/v5/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetCommit/v5/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetFetch/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetFetch/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetFetch/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetFetch/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetFetch/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetFetch/v3/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetFetch/v3/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetFetch/v4/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/offsetFetch/v4/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v3/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v3/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v4/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v4/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v5/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v5/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v6/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v6/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v7/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/produce/v7/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/saslAuthenticate/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/saslHandshake/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/saslHandshake/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/saslHandshake/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/saslHandshake/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/saslHandshake/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/syncGroup/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/syncGroup/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/syncGroup/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/syncGroup/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/syncGroup/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/syncGroup/v2/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/syncGroup/v2/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/syncGroup/v3/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/syncGroup/v3/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/index.js create mode 100644 node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v0/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v0/response.js create mode 100644 node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v1/request.js create mode 100644 node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v1/response.js create mode 100644 node_modules/kafkajs/src/protocol/resourcePatternTypes.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/awsIam/index.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/awsIam/request.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/awsIam/response.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/oauthBearer/index.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/oauthBearer/request.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/oauthBearer/response.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/plain/index.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/plain/request.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/plain/response.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/scram/finalMessage/request.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/scram/finalMessage/response.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/scram/firstMessage/request.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/scram/firstMessage/response.js create mode 100644 node_modules/kafkajs/src/protocol/sasl/scram/index.js create mode 100644 node_modules/kafkajs/src/protocol/timestampTypes.js create mode 100644 node_modules/kafkajs/src/retry/defaults.js create mode 100644 node_modules/kafkajs/src/retry/defaults.test.js create mode 100644 node_modules/kafkajs/src/retry/index.js create mode 100644 node_modules/kafkajs/src/utils/arrayDiff.js create mode 100644 node_modules/kafkajs/src/utils/groupBy.js create mode 100644 node_modules/kafkajs/src/utils/lock.js create mode 100644 node_modules/kafkajs/src/utils/long.js create mode 100644 node_modules/kafkajs/src/utils/mapValues.js create mode 100644 node_modules/kafkajs/src/utils/once.js create mode 100644 node_modules/kafkajs/src/utils/seq.js create mode 100644 node_modules/kafkajs/src/utils/sharedPromiseTo.js create mode 100644 node_modules/kafkajs/src/utils/shuffle.js create mode 100644 node_modules/kafkajs/src/utils/sleep.js create mode 100644 node_modules/kafkajs/src/utils/swapObject.js create mode 100644 node_modules/kafkajs/src/utils/uniq.js create mode 100644 node_modules/kafkajs/src/utils/waitFor.js create mode 100644 node_modules/kafkajs/src/utils/websiteUrl.js create mode 100644 node_modules/kafkajs/types/index.d.ts create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 src/apps/dummy-consumer.mjs create mode 100644 src/apps/near-intents-ingest.mjs create mode 100644 src/bus/kafka.mjs create mode 100644 src/bus/kafka/consumer.mjs create mode 100644 src/bus/kafka/producer.mjs create mode 100644 src/core/env.mjs create mode 100644 src/core/event-envelope.mjs create mode 100644 src/core/log.mjs create mode 100644 src/core/pair-filter.mjs create mode 100644 src/lib/config.mjs create mode 100644 src/lib/env.mjs create mode 100644 src/lib/event-envelope.mjs create mode 100644 src/venues/near-intents/ingest.mjs create mode 100644 src/venues/near-intents/normalize.mjs create mode 100644 src/venues/near-intents/ws.mjs diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..8dfd1e9 --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +NEAR_INTENTS_API_KEY=your_solver_jwt +NEAR_INTENTS_WS_URL=wss://solver-relay-v2.chaindefuser.com/ws +KAFKA_BROKERS=127.0.0.1:9092 +KAFKA_CLIENT_ID=trading-system +KAFKA_TOPIC_NORM_SWAP_DEMAND=norm.swap_demand +KAFKA_CONSUMER_GROUP_DUMMY=dummy-reactor-v1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c1e8ad7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.ant-colony/ +.venv/ +__pycache__/ +*.pyc +.env diff --git a/README.md b/README.md new file mode 100644 index 0000000..cac6315 --- /dev/null +++ b/README.md @@ -0,0 +1,134 @@ +# near-intents-monitor + +Minimal event-driven POC for the first trading-system component: + +- **venue ingest**: NEAR Intents solver-bus quote flow +- **central bus**: Redpanda / Kafka-compatible broker +- **dummy reactor**: placeholder consumer for later trade-decision logic + +## Architecture + +```text +NEAR Intents WebSocket + | + v +src/apps/near-intents-ingest.mjs + | + +--> raw.near_intents.quote + | + +--> norm.swap_demand + | + v + src/apps/dummy-consumer.mjs +``` + +The ingest app connects to the NEAR Intents websocket, subscribes to `quote` and `quote_status`, normalizes quote demand, and publishes to a Kafka-compatible topic. + +## Project structure + +```text +src/ + apps/ + near-intents-ingest.mjs + dummy-consumer.mjs + bus/ + kafka/ + producer.mjs + consumer.mjs + core/ + event-envelope.mjs + log.mjs + pair-filter.mjs + lib/ + env.mjs + config.mjs + venues/ + near-intents/ + ingest.mjs + normalize.mjs + ws.mjs +``` + +## Environment + +Create `.env` in repo root: + +```env +NEAR_INTENTS_API_KEY=your_solver_jwt +NEAR_INTENTS_WS_URL=wss://solver-relay-v2.chaindefuser.com/ws +KAFKA_BROKERS=127.0.0.1:9092 +KAFKA_CLIENT_ID=trading-system +KAFKA_TOPIC_NORM_SWAP_DEMAND=norm.swap_demand +KAFKA_CONSUMER_GROUP_DUMMY=dummy-reactor-v1 +``` + +### Broker notes + +- `KAFKA_BROKERS` accepts a comma-separated broker list. +- Redpanda works because the apps use the Kafka protocol via `kafkajs`. +- `src/lib/config.mjs` is the shared config loader for both app entrypoints. +- The ingest app publishes normalized quote-demand events to `norm.swap_demand` by default. + +## Install + +```bash +npm install +``` + +## Run + +### Start NEAR Intents ingest + +Use the package script: + +```bash +npm run near-intents:ingest +``` + +Or run the app directly: + +```bash +node src/apps/near-intents-ingest.mjs +``` + +Optional exact-pair filter: + +```bash +npm run near-intents:ingest -- --pair 'asset_a->asset_b' +``` + +Example: + +```bash +npm run near-intents:ingest -- --pair 'nep141:btc.omft.near->nep141:gnosis-0x420ca0f9b9b604ce0fd9c18ef134c705e5fa3430.omft.near' +``` + +The filter is direction-agnostic, so `asset_a->asset_b` also matches `asset_b->asset_a`. + +### Start the dummy consumer + +Use the package script: + +```bash +npm run dummy-consumer +``` + +Or run the app directly: + +```bash +node src/apps/dummy-consumer.mjs +``` + +The dummy consumer subscribes to `norm.swap_demand`, logs the observed pair and quote id, and stands in for a future decision engine. + +## Scripts + +- `npm run near-intents:ingest` — start the websocket ingest and publish to Kafka/Redpanda topics +- `npm run dummy-consumer` — consume normalized demand events +- `npm start` — legacy wrapper that forwards into the ingest app + +## Notes + +- This repo is now bus-first: venue intake and downstream reaction are decoupled through Kafka-compatible topics. +- `index.mjs` remains only as a compatibility launch wrapper; operational docs should prefer `src/apps/*` entrypoints and npm scripts. +- Older single-file, Python, or TUI-only runtime instructions are obsolete for this repository state. diff --git a/docs/minimal-product.md b/docs/minimal-product.md new file mode 100644 index 0000000..6de4003 --- /dev/null +++ b/docs/minimal-product.md @@ -0,0 +1,198 @@ +# Minimal product: NEAR Intents demand monitor + +## Goal +Build the smallest useful event-driven product for crypto trading research: + +- read **live user demand** from NEAR Intents +- publish demand into a **central Kafka/Redpanda-compatible bus** +- prove downstream consumption with a **dummy reactor** +- avoid dashboards, execution, wallets, storage, auth workflows beyond the required API key, strategy code, and generic infra beyond the message bus itself + +## Why this is the right first slice +From the NEAR Intents docs, there are several possible data surfaces: + +1. **Message Bus WebSocket `quote` subscription** + - Endpoint: `wss://solver-relay-v2.chaindefuser.com/ws` + - Real-time stream for quote requests + - Subscription request shape: + ```json + { + "jsonrpc": "2.0", + "id": 1, + "method": "subscribe", + "params": ["quote"] + } + ``` + - Expected live frame shape is JSON-RPC-like but should be treated as flexible. The adapter should accept quote payloads when the useful fields appear either: + - directly under `params` + - directly under `result` + - or at the top level of the message body + - Fields of interest include: + - `quote_id` (or equivalent request identifier) + - `defuse_asset_identifier_in` + - `defuse_asset_identifier_out` + - `exact_amount_in` or `exact_amount_out` + - `min_deadline_ms` + - Subscription acknowledgements may also vary. They may arrive as an `id`-matched JSON-RPC response with a simple `result`, a structured `result`, or other non-quote control frame before the first quote event. + - This is the closest public signal to **current demand**. + +2. **Message Bus JSON-RPC `publish_intent` / `get_status`** + - Endpoint: `https://solver-relay-v2.chaindefuser.com/rpc` + - Useful for posting intents or checking a known `intent_hash` + - Not a public firehose of all intents. + +3. **Explorer API `/api/v0/transactions`** + - Historical and analytics friendly + - Requires JWT auth + - Better for history, not best for a minimal live monitor + +4. **Verifier contract intent payloads** + - The on-chain swap expression is usually `token_diff` + - Important for understanding settlement semantics + - Not the easiest first live intake path for a lean bus-first system + +## Product decision +The minimal product should monitor **WebSocket `quote` events** and route them through a bus-first runtime. + +### Why +- closest live signal to user demand +- directly reflects what users are requesting from solvers +- enough to answer the first trading question: **what assets are being requested right now?** +- decouples venue intake from downstream analysis through Kafka-compatible topics + +### Important implementation note +Current docs for the market-maker quickstart and live endpoint behavior indicate the Message Bus requires a **partner API key / JWT** in the `Authorization: Bearer ...` header. +That means the best path is still the quote stream, but live operation is partner-gated. + +### Important caveat +A `quote` event is **pre-trade demand**, not guaranteed execution. +That is fine for v0. The purpose is demand sensing, not settlement accounting. + +## Runtime shape + +```text +NEAR Intents websocket + | + v +src/apps/near-intents-ingest.mjs + | + +--> raw.near_intents.quote + | + +--> norm.swap_demand + | + v + src/apps/dummy-consumer.mjs +``` + +### Runtime contracts + +#### Ingest app +`src/apps/near-intents-ingest.mjs`: +- loads env +- parses optional `--pair 'asset_a->asset_b'` +- starts the NEAR Intents websocket adapter +- writes raw and normalized events to the configured broker + +#### Dummy consumer +`src/apps/dummy-consumer.mjs`: +- subscribes to `norm.swap_demand` +- logs observed pair and quote id +- exists only to prove a downstream consumer contract + +#### Bus config +Default env-driven topics and group ids: +- `KAFKA_TOPIC_RAW_NEAR_INTENTS_QUOTE=raw.near_intents.quote` +- `KAFKA_TOPIC_NORM_SWAP_DEMAND=norm.swap_demand` +- `KAFKA_CONSUMER_GROUP_DUMMY=dummy-reactor-v1` + +Redpanda is a valid runtime target because the transport is Kafka-compatible. + +## Internal model +Normalize each quote event into a thin bus envelope: + +Top-level envelope fields: +- `venue` +- `source` +- `type` +- `eventId` +- `occurredAt` +- `ingestedAt` +- `assetIn` +- `assetOut` +- `raw` +- `quote` + +Nested `quote` fields: +- `quoteId` +- `assetIn` +- `assetOut` +- `amountIn` +- `amountOut` +- `ttlMs` + +Field extraction must remain tolerant to known upstream aliases, and normalization should continue to operate on the merged `metadata + data` payload shape from the Message Bus event. +The live adapter now intentionally accepts quote-like payloads from `params`, `result`, or the top-level message body, but only processes frames that actually look like quote data. Subscription acknowledgements and unrelated control frames should still be ignored. + +## Filtering +The ingest runtime supports an optional exact-pair filter: + +```bash +npm run near-intents:ingest -- --pair 'asset_a->asset_b' +``` + +The filter is direction-agnostic, so the reversed asset order is also accepted. + +## Scope boundaries +### Must do +- connect to the websocket +- subscribe to `quote` and tolerate control frames +- normalize quote events into one compact model +- publish raw and normalized events to Kafka/Redpanda-compatible topics +- allow a downstream consumer to react to normalized events +- reconnect automatically on disconnect +- document `npm` and `node` entrypoints + +### Must not do +- Python packaging or CLI guidance +- TUI-specific product requirements +- charts +- account details +- pnl +- routing internals +- market making controls +- execution buttons +- config panels +- speculative infra beyond the current bus and dummy consumer + +## Path to success +1. Connect to WebSocket +2. Subscribe to `quote` +3. Normalize incoming events into one compact model +4. Publish raw envelopes to `raw.near_intents.quote` +5. Publish normalized envelopes to `norm.swap_demand` +6. Start a dummy consumer on the normalized topic +7. Reconnect automatically on disconnect +8. Only after this works, consider: + - `quote_status`-specific downstream handling + - historical replay via Explorer API + - token metadata enrichment + - filtering and alerts beyond `--pair` + +## Packaging alignment +Current repository packaging and usage should stay aligned around the JavaScript runtime entrypoints: + +- package scripts: + - `npm run near-intents:ingest` + - `npm run dummy-consumer` + - `npm start` as a compatibility wrapper +- direct app entrypoints: + - `node src/apps/near-intents-ingest.mjs` + - `node src/apps/dummy-consumer.mjs` + +Documentation should treat the npm scripts and `src/apps/*` node entrypoints as canonical. Older single-file and Python/TUI instructions should remain removed to avoid runtime confusion. + +## Sources +- NEAR Intents Message Bus WebSocket docs: `subscribe` with `quote` / `quote_status` +- NEAR Intents Message Bus RPC docs: `quote`, `publish_intent`, `get_status` +- Verifier contract docs: `token_diff` intent type +- Explorer API OpenAPI: authenticated historical transactions diff --git a/docs/spec.md b/docs/spec.md new file mode 100644 index 0000000..71be2de --- /dev/null +++ b/docs/spec.md @@ -0,0 +1,144 @@ +# NEAR Intents demand monitor: bus-first source plan + +## Why websocket quote requests are still the MVP demand signal + +Public solver quote requests remain the closest thing to live demand because they appear when a user or integration asks the network for executable pricing. They are still the right upstream source, but the runtime architecture is now bus-first rather than terminal-first. + +Why this source wins for a first monitor: + +- **Most real-time:** quote requests arrive before settlement and usually before a completed trade is visible anywhere else. +- **Closer to intent formation:** they reflect active user demand, not just historical outcomes. +- **Operationally simple:** a single websocket feed can drive the ingest side without indexing chains, scraping dashboards, or correlating multiple APIs. +- **Good enough for ranking demand:** even if quotes do not always become fills, repeated quote flow is still a strong indicator of what users are currently trying to do. + +## Tradeoffs vs other sources + +### Solver websocket quote requests + +Pros: +- lowest-latency view of current demand +- directly tied to solver workflow +- suitable for a streaming ingest adapter +- can be normalized into pair, size, and frequency metrics immediately + +Cons: +- quote requests are **interest**, not guaranteed executed volume +- public access may still be rate-limited, undocumented, or require credentials depending on environment +- schema and availability may change faster than user-facing products + +### Explorer + +Explorer (`https://explorer.near-intents.org/`) is useful for validation and historical inspection, but it is usually a worse primary source for an MVP demand monitor. + +Tradeoffs: +- better for human inspection than low-latency streaming +- likely shows processed/published activity instead of raw quote demand +- may lag the actual request path +- less convenient as a machine-first demand feed + +### Status dashboard / published status + +Status (`https://status.near-intents.org/posts/dashboard`) is useful for system health, not demand discovery. + +Tradeoffs: +- tells us whether the platform is up, degraded, or incident-affected +- does **not** represent per-request user demand +- coarse and aggregated by design + +### Published intents / settled outcomes + +Published or completed intents are higher-confidence signals, but lower-fidelity for immediate demand sensing. + +Tradeoffs: +- stronger evidence of actual execution +- misses abandoned demand and pre-trade discovery +- arrives later than quote traffic +- may require more indexing and entity correlation work + +## Runtime architecture + +```text +solver websocket quote stream + | + v +src/apps/near-intents-ingest.mjs + | + +--> raw.near_intents.quote + | + +--> norm.swap_demand + | + v + src/apps/dummy-consumer.mjs +``` + +### Responsibilities + +#### `src/apps/near-intents-ingest.mjs` +- loads env from `.env` +- parses optional `--pair 'asset_a->asset_b'` +- connects to the NEAR Intents websocket +- subscribes to `quote` and `quote_status` +- publishes raw venue envelopes to `raw.near_intents.quote` +- publishes normalized swap-demand envelopes to `norm.swap_demand` + +#### `src/apps/dummy-consumer.mjs` +- consumes normalized events from `norm.swap_demand` +- logs observed demand as a placeholder for later strategy logic + +#### Kafka / Redpanda layer +- broker endpoint comes from `KAFKA_BROKERS` +- Redpanda is supported through Kafka protocol compatibility +- topics are configurable via env and default to: + - `raw.near_intents.quote` + - `norm.swap_demand` + +## Assumptions and limitations + +- The websocket is the best available **MVP** source, not a perfect truth source. +- Demand is approximated by quote requests, not by settled intents. +- Live endpoints require auth in practice; `NEAR_INTENTS_API_KEY` must be provided. +- Request schemas may evolve; the parser should tolerate missing fields. +- The current product is intentionally minimal: no database, no backfill, no reconciliation against chain state. +- The dummy consumer proves the decoupled flow but is not a strategy engine. + +## Run instructions + +Install: + +```bash +npm install +``` + +Start ingest: + +```bash +npm run near-intents:ingest +``` + +Direct node entrypoint: + +```bash +node src/apps/near-intents-ingest.mjs +``` + +Run with exact-pair filtering: + +```bash +npm run near-intents:ingest -- --pair 'asset_a->asset_b' +``` + +Start dummy consumer: + +```bash +npm run dummy-consumer +``` + +Direct node entrypoint: + +```bash +node src/apps/dummy-consumer.mjs +``` + +## Decision summary + +For an MVP whose job is to answer "what are users asking for right now?", solver websocket quote requests are still the best first source because they are the most direct, timely, and stream-friendly signal. The implementation now routes that signal through Kafka/Redpanda topics so ingestion and downstream reaction can evolve independently. diff --git a/index.mjs b/index.mjs new file mode 100644 index 0000000..bd207a7 --- /dev/null +++ b/index.mjs @@ -0,0 +1 @@ +import './src/apps/near-intents-ingest.mjs'; diff --git a/node_modules/.package-lock.json b/node_modules/.package-lock.json new file mode 100644 index 0000000..d36d3c7 --- /dev/null +++ b/node_modules/.package-lock.json @@ -0,0 +1,17 @@ +{ + "name": "near-intents-monitor-poc", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "node_modules/kafkajs": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-2.2.4.tgz", + "integrity": "sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + } + } +} diff --git a/node_modules/kafkajs/CHANGELOG.md b/node_modules/kafkajs/CHANGELOG.md new file mode 100644 index 0000000..fbeda3a --- /dev/null +++ b/node_modules/kafkajs/CHANGELOG.md @@ -0,0 +1,754 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). + +## [2.2.4] - 2023-02-27 + +### Added + - Include `groupId` in debug log when failing to find group coordinator #1522 + +### Fixed + - Rejoin group after ILLEGAL_GENERATION error #1474 + - Fix consumer getting stuck after very brief throttling #1532 + - Prevent infinite crash loop when no brokers are available #1408 + +## [2.2.3] - 2022-11-21 + +### Fixed + - Fix regression in SASL/PLAIN authentication #1480 + +## [2.2.2] - 2022-10-18 + +### Fixed + - Fix compatibility with AWS MSK Serverless #1463 + +## [2.2.1] - 2022-10-13 + +### Fixed + - Fixed Typescript definitions for custom authentication mechanisms #1459 + +## [2.2.0] - 2022-08-16 + +### Added + - Add the ability to inject custom authentication mechanisms #1372 + - Add admin methods `alterPartitionReassignments` & `listPartitionReassignments` #1419 + +### Fixed + - Fix deprecation warning when connecting to a broker over TLS via IP address #1425 + - Improve consumer performance when subscribed to thousands of topics #1436 + +## [2.1.0] - 2022-06-28 + +### Added + - Add `pause` function to `eachMessage`/`eachBatch` to pause the current topic-partition #1364 + - The `KafkaMessage` type is now a union between the pre-Kafka 0.10 message format and the current #1401 + +### Fixed + - Fix 100% CPU utilization when all brokers are unavailable #1402 + - Fix persistent error when trying to produce after a topic authorization error #1385 + - Fix error when aborting or committing an empty transaction #1388 + - Don't re-process messages from a paused partition after breaking the consumption flow #1382 + +## [2.0.2] - 2022-05-31 + +### Fixed + - Fix `consumer.seek` when seeking on multiple partitions for the same topic #1378 + +## [2.0.1] - 2022-05-23 + +### Fixed + - Fix members leaving the group after not being assigned any partitions #1362 + - Make `REPLICA_NOT_AVAILABLE` retriable #1351 + - Document `admin.createTopics` respecting cluster default partitions number and replication factor #1360 + +## [2.0.0] - 2022-05-06 + +This is the first major version released in 4 years, and contains a few important breaking changes. **A [migration guide](https://kafka.js.org/docs/migration-guide-v2.0.0) has been prepared to help with the migration process.** Be sure to read it before upgrading from older versions of KafkaJS. + +### Added + - Validate configEntries when creating topics #1309 + - New `topics` argument for `consumer.subscribe` to subscribe to multiple topics #1313 + - Support duplicate header keys #1132 + +### Removed + - **BREAKING:** Drop support for Node 10 and 12 #1333 + - **BREAKING:** Remove deprecated enum `ResourceTypes` #1334 + - **BREAKING:** Remove deprecated argument `topic` from `admin.fetchOffsets` #1335 + - **BREAKING:** Remove deprecated method `getTopicMetadata` from admin client #1336 + - **BREAKING:** Remove typo type `TopicPartitionOffsetAndMedata` #1338 + - **BREAKING:** Remove deprecated error property originalError. Replaced by `cause` #1341 + +### Changed + - **BREAKING:** Change default partitioner to Java compatible #1339 + - Improve consumer performance #1258 + - **BREAKING:** Enforce request timeout by default #1337 + - **BREAKING** Honor default replication factor and partition count when creating topics #1305 + - Increase default authentication timeout to 10 seconds #1340 + +### Fixed + - Fix invalid sequence numbers when producing concurrently with idempotent producer #1050 #1172 + - Fix correlation id and sequence number overflow #1310 + - Fix consumer not restarting on retriable connection errors #1304 + - Avoid endless sleep loop #1323 + +## [1.16.0] - 2022-02-09 + +### Added + - Allow manual heartbeating from inside `eachMessage` handler #1255 + - Add `rebalancing` consumer event #1067 #1079 + - Add overload typings for all event types #1202 + - Return `configSource` in `admin.decribeConfigs` #1023 + - Add `topics` property to `admin.fetchOffsets` to fetch offsets for multiple topics #992 #998 + - Improve error output from `admin.createTopic` #1104 + - Export Error classes #1254 + - Validate `brokers` list contains strings #1284 + - Throw error when failing to stop or disconnect consumer #960 + +### Changed + + - Don't commit offsets from `consumer.seek` when `autoCommit` is `false` #1012 + - Do not restart the consumer on non-retriable errors #1274 + - Downgrade consumer rebalance error log to `warn` #1279 + - Make default round-robin partitioner topic-aware #1112 + +### Fixed + - Fix `offset` type of `consumer.seek` #981 + - Fix crash when used in Electron app built with electron-builder #984 + - Improve performance of Fetch requests #985 + - Fix crash when using topics with name of built-in Javascript functions #995 + - Fix type of consumer constructor to require config object #1002 + - Fix message type to allow `null` key #1037 + - Respect `heartbeatInterval` when invoking `heartbeat` concurrently #1026 + - Fix type of `timestamp` of `LoggerEntryContent` to be string #1082 + - Fix return type of `admin.describeAcls` #1118 + - Fix consumer getting stuck in `DISCONNECTING` state if in-flight requests time out during disconnect #1208 + - Fix failed serialization of BigInts when logging #1234 + - Fix crash when committing offsets for a topic before consumer initialization #1235 + - Reauthenticate to all brokers on demand #1241 + - Remove unnecessary warn log when calling `admin.deleteTopicRecords` with offset `-1` #1265 + - Handle empty control batches #1256 + - Send empty topic array as null when fetching metadata #1184 + +## [1.15.0] - 2020-11-24 +### Added + - Initial work for static membership #888 + - Add consumer instrumentation event: received unsubscribed topics #897 + - Add option for `admin.fetchOffsets` to resolve the offsets #895 + - Add ACL functions to admin client #697 + - Add `admin.deleteTopicRecords` #905 + - Emit `GROUP_JOIN` event on stale partition assignments #937 + +### Changed + - Added properties to error classes typescript types #900 + - Make header value type definition possibly undefined #927 + - Bump API versions for client-side throttling #933 + - Add `UNKNOWN_TOPIC_OR_PARTITION` check for `addMultipleTargetTopics` #938 + +### Fixed + - Fix describe/alter broker configs (introduced `ConfigResourceTypes`) #898 + - Fix record batch compression masking (fix ZSTD compression) #912 + - Prevent inflight's correlation id collisions #926 + - Fix ACL, ISocketFactory and SaslOptions type definitions #941 #959 #966 + - Fix deadlock on the connection `onError` handler #944 + - Fix deadlock on the connection `onTimeout ` handler #956 + - Remove nested retriers from producer #962 (fixes #958 #950) + +## [1.14.0] - 2020-09-21 +### Added + - Support Produce v6 protocol #869 + - Support Produce v7 protocol (support for ZSTD compression) #869 + - Broker rediscovery with config.brokers parameter taking a callback function #854 #882 + +### Changed + - Remove long.js in favor of BigInt #663 + - Remove allowExperimentalV011 flag #847 + +### Fixed + - Only commit offsets on eachMessage failures if autoCommit is enabled #866 + - Fix consumer offsets not committed if consumer stop was invoked right after the batch process #874 + - Remove brokers with closed connections from the brokers list #878 + - Type improvements and fixes #877 + +## [1.13.0] - 2020-09-10 +### Added + - Add listGroup method to admin interface #645 + - Add describeCluster method to admin client #648 + - Add createPartitions method to admin client #661 + - Add deleteGroups method to admin client #646 + - Add listTopics method to admin client #718 + - Add describeGroups method to admin client #742 + - Allow to handle consumer retry failure at the user level #643 + - Support Fetch v8 protocol (including client-side throttling) #776 + - Support Fetch v9 protocol #778 + - Support Fetch v10 protocol #792 + - Support Fetch v11 protocol #810 + - Support JoinGroup v3 and v4 protocol #801 + - Oauthbearer support #680 + - Add new protocol errors #824 + - Add versioning to docs #835 + - Add fetch topic offsets by timestamp #604 + - Support LOG_APPEND_TIME record timestamps #838 + - Suppress JoinGroup V4+ response error log when memberId is empty #860 + +### Changed + - Replace fetch promise all with async generator #570 + - Improve balance in the RoundRobinAssigner #635 + - Add single requestTimeout runner instead of setTimeout per request #650 + - Provide the subscribed topics to the protocol() function #545 + - Only wait for the lock when there are enqueued batches #670 + - Update consumer default retries to 5 #720 (related to #719) + - Simplify and speed up SeekOffsets #668 + - Remove maxInFlight option from default retry and moved into Producer ad Consumer #754 + - Use addMultipleTargetTopics instead of looping over multiple calls to addTargetTopic #748 + - Only disconnect the consumer and producers if they were created #784 + - Resolve socket requests without response immediately when they have been queued #785 + - Move default request timeout from connection to cluster #739 + - Simplify the BufferedAsyncIterator #671 + - Ensure fair fetch response allocation across topic-partitions #859 + +### Fixed + - Type improvements and fixes #636 #664 #675 #722 #729 #758 #799 #813 #757 #749 #764 #828 #843 #839 + - Remove invalid topics from targetTopics on INVALID_TOPIC_EXCEPTION #666 + - Fix network buffering performance problems #669 + - Delete the entry for the waiter when the timeout is reached #694 + - Fix encoder instanceof issue with Encoder #685 + - Fix default retry for consumer #719 + - Runner#waitForConsumer uses consuming stop event instead of timers #724 + - Fix unhandled rejections #714 #797 + - Make Array shuffle test pass in Node >= 11.0.0 #740 + - Use setImmediate when scheduling calls of scheduleFetch #752 + - Improve offset commit handling #775 + - Use the length from the message to pre-allocate the result array #771 + - Fix application lock in case of errors before the connection is established #780 + - Add isNaN check for concurrency limit #787 + - Fix admin client createTopics timeout #800 + - Avoid repeated costly copies of buffers when working with encoders #811 + - Fix fall-back retry config for producer #851 + +## [1.12.0] - 2020-01-30 +### Added + - Force refresh of metadata when topic is not present #556 + - Expose ConsumerRunConfig type #615 + - Randomize order of seed brokers #632 + +### Changed + - Support TLS SNI by default #512 + - Changed typing of `logLevel` argument of `logCreator` #538 + - Add type boolean in ssl KafkaConfig #557 + - Allow logging Fetch response payload buffers #573 + - Remove default null for logCreator #595 + - Add error names to ensure error names work with webpack + uglify #602 + - Merge TopicMessages by topic in producer sendBatch #626 + +### Fixed + - Skip control records without auto-resolve #511 + - Handle empty member assignment #567 + - Only fetch for partitions with initialized offsets #582 + - Get correct next offset for compacted topics #577 + - TS type definition for removing instrumentation event listeners #608 + - Fixed IHeaders definition to accept plain strings #547 + - Make TS type ProducerBatch fields optional #610 + - Fix typings for logger getters #620 + +## [1.11.0] - 2019-09-30 +### Added + - Add Typescript SASLMechanism type to definitions #477 + - Allow SASL Connections to Periodically Re-Authenticate #496 + +### Changed + - Throw validation error when the broker list is empty #460 + - Improve the encoder to avoid copying unnecessary bytes #471 + - Throw an error on subscription changes for running consumers #470 + - Default `throttle_time_ms` to 0 if missing in `ApiVersions` response #495 + - Remove normalisation of the password when using SCRAM mechanism #505 + +### Fixed + - Fix built-in partitioners type definition error #455 + - Detect replaced brokers on refresh metadata #457 + - Make NodeJS REPL get correct `randomBytes()` #462 + - Fix `IHeaders` type definition (from string to Buffer) #467 + - Rename Typescript definitions from `ResourceType` to `ResourceTypes` #468 + - Update Typescript definitions to make `configNames` optional in `ResourceTypes` #474 + - Fix `transactionState.ABORTING` value #478 + +## [1.10.0] - 2019-07-31 +### Added + - Allow the consumer to pause and resume individual partitions #417 + - Add `consumer.commitOffsets` for manual committing #436 + - Expose consumer paused partitions #444 + +### Changed + - Removed unnecessary async code #434 + - Use SubscriptionState to track member assignment #429 + +### Fixed + - Improve type compatibility with @types/kafkajs #416 + - Fix `fetchTopicMetadata` return type #433 + +## [1.9.3] - 2019-06-27 +### Fixed + - Fix AWS-IAM mechanism name #411 #412 + - Fix TypeScript types for topic subscription with RegExp #413 + +## [1.9.2] - 2019-06-26 +### Fixed + - Fix typescript types for Logger, consumer pause and resume, eachMessage and EachBatch interfaces #409 + +## [1.9.1] - 2019-06-25 +### Fixed + - Fix typescript types for SSL, SASL and batch #407 + +## [1.9.0] - 2019-06-25 +### Added + - Add typescript declaration file #362 #385 #390 + - Add `requestTimeout` to apiVersions #369 + - Discard messages saw a seek operation during a fetch or batch processing #367 + - Include fetched offset metadata retrieved with `admin.fetchOffsets` #389 + - Allow offset metadata to be written as part of OffsetCommit requests #392 + - Prevent the consumption of messages for topics paused while fetch is in-flight #397 + - Add `AWS-IAM` SASL mechanism #402 + - Add `batch.offsetLagLow` #405 + +### Changed + - Don't modify `global.crypto` #365 + - Change log level about producer without metadata #382 + - Update encoder to write arrays as single `Buffer.concat` where possible #394 + +### Fixed + - Log error message on connection errors #400 + - Make sure runner has connected brokers and fresh metadata before it starts #404 + +## [1.8.1] - 2019-06-25 +### Fixed + - Make sure runner has connected brokers and fresh metadata before it starts #404 + +## [1.8.0] - 2019-05-13 +### Added + - Add partition-aware concurrent mode for `eachMessage` #332 + - Add `JavaCompatiblePartitioner` #358 + - Add `consumer.subscribe({ topic: RegExp })` #346 + - Update supported protocols to latest of Kafka 1 #343 #347 #348 + +### Changed + - Add documentation link to `REBALANCE_IN_PROGRESS` error #341 + +### Fixed + - Fix crash on offline replicas in metadata v5 response #350 + +## [1.7.0] - 2019-04-12 +### Fixed + - Improve compatibility with terserjs #338 + +### Added + - Add `admin#fetchTopicMetadata` #331 + +### Changed + - Deprecated `admin#getTopicMetadata` #331 + - `admin#fetchTopicOffsets` returns the low and high watermarks #333 + +## [1.6.0] - 2019-04-01 +### Added + - Allow providing a socketFactory on client creation #263 + - Add fetchTopicOffsets method #314 + +## [1.5.2] - 2019-04-01 +### Fixed + - Process a fixed number of lock releases per iteration on `lock#release` #323 + +### Changed + - Use the max between the default request timeout and the protocol override #318 + - Only emit events if there are listeners #321 + +## [1.5.1] - 2019-03-14 +### Fixed + - Handle `null` keys on isAbortMarker #312 + +### Changed + - Prevent subsequent calls to `consumer#run` to override the running consumer #305 + - Improve browser compatibility #300 + - Add custom `requestTimeout` for protocol fetch #310 + - Make `requestTimeout` optional, the current implementation is behind the flag `enforceRequestTimeout` #313 + +## [1.5.0] - 2019-03-05 +### Changed + - See `1.5.0-beta.X` versions + +## [1.5.0-beta.4] - 2019-02-28 +### Fixed + - Abort old transactions on protocol error `CONCURRENT_TRANSACTIONS` #299 + +## [1.5.0-beta.3] - 2019-02-20 +### Fixed + - Missing default restart time on crashes due to retriable errors #283 + - Add custom requestTimeout for JoinGroup v0 #293 + - Fix propagation of custom retry configs #295 + +### Changed + - Allow calling `Producer.sendBatch` with empty list #287 + - Encode non-buffer key as string by default #291 + +## [1.5.0-beta.2] - 2019-02-13 +### Fixed + - Handle undefined message key when producing with 0.11 #247 + - Fix consumer restart on find coordinator errors #253 + - Crash consumer on codec not implemented error #256 + - Fix error message on invalid username or password #270 + - Restart consumer on crashes due to retriable error #269 + - Remove deleted topics from the cluster target group #273 + +### Changed + - Change Node engine requirement to >=8.6.0 #250 + - Don't include lockfile and vscode files in package #264 + +### Added + - Allow configuring log level at runtime #278 + +## [1.5.0-beta.1] - 2019-01-17 +### Fixed + - Rolling upgrade from 0.10 to 0.11 causes unknown magic byte errors #246 + +### Changed + - Validate consumer groupId #244 + +### Added + - Expose network queue size event to consumers, producers and admin #245 + +## [1.5.0-beta.0] - 2019-01-08 +### Changed + - Add transactional attributes to record batch #199 + - Ignore control records #208 + - Filter aborted messages on the consumer #223 #210 #228 + - Make Round robin assigner forward `userdata` #231 + +### Added + - Protocol `FindCoordinator` v1 #189 + - Protocol `InitProducerId` v0 #190 + - Protocol `AddPartitionsToTxn` v0 #191 + - Protocol `AddOffsetsToTxn` v0 #194 + - Protocol `TxnOffCommit` v0 #195 + - Protocol `EndTxn` v0 #198 + - Protocol `ListOffsets` v1 and v2 #217 #209 + - Accept max in-flight requests on the connection #216 + - Idempotent producer #203 + - Transactional producer #206 + - Protocol `SASLAuthenticate` #229 + - Add SendOffsets to consumer `eachBatch` #232 + - Add network instrumentation events #233 + - Allow users to provide offsets to the `commitOffsetsIfNecessary` #235 + +## [1.4.8] - 2019-02-18 +### Fixed + - Handle undefined message key when producing with 0.11 #247 + - Fix consumer restart on find coordinator errors #253 + - Crash consumer on codec not implemented error #256 + - Fix error message on invalid username or password #270 + +## [1.4.7] - 2019-01-17 +### Fixed + - Rolling upgrade from 0.10 to 0.11 causes unknown magic byte errors #246 + +## [1.4.6] - 2018-12-03 +### Fixed + - Always assign partitions based on subscribed topics #227 + +## [1.4.5] - 2018-11-28 +### Fixed + - Fix crash in mitigation for receiving metadata for unsubscribed topics #221 + +### Added + - Add `CRASH` instrumentation event for the consumer #221 + +## [1.4.4] - 2018-10-29 +### Fixed + - Protocol produce v3 wasn't filtering `undefined` timestamps and was sending timestamp 0 (`NaN` converted) for all messages #188 + +## [1.4.3] - 2018-10-22 +### Changed + - Version `1.4.2` without test files + +## [1.4.2] - 2018-10-22 +### Changed + - Allow messages with a value of `null` to support tombstones #185 + +## [1.4.1] - 2018-10-17 +### Fixed + - Decode multiple RecordBatch on protocol Fetch v4 #179 + - Skip incomplete record batches #182 + - Producer with `acks=0` never resolve #181 + +### Added + - Runtime flag for displaying buffers in debug output #176 + - Add ZSTD to compression codecs and types #157 + - Admin get topic metadata #174 + +### Changed + - Add description to lock instances #178 + +## [1.4.0] - 2018-10-09 +### Fixed + - Potential offset loss when updating offsets for resolved partitions #124 + - Refresh metadata on lock timeout #131 + - Cleans up stale brokers on metadata refresh #131 + - Force metadata refresh on `ECONNREFUSED` #134 + - Handle API version not supported #135 + - Handle v0.10 messages on v0.11 Fetch API #143 + +### Added + - Admin delete topics #117 + - Update metadata api and allow to disable auto topic creation #118 + - Use highest available API version #135 #146 + - Admin describe and alter configs #138 + - Validate message format in producer #142 + - Consumers can detect that a topic was updated and force a rebalance #136 + +### Changed + - Improved stack trace for `KafkaJSNumberOfRetriesExceeded` #123 + - Enable Kafka v0.11 API by default #141 (Can still be disabled with `allowExperimentalV011=false`) + - Replace event emitter Lock #154 + - Add member assignment to `GROUP_JOIN` instrumentation event #136 + +## [1.3.1] - 2018-08-20 +### Fixed + - Client logger accessor #106 + - Producer v3 decode format #114 + - Parsing multiple responses #115 + - Fetch v4 for partial messages on record batch #116 + +### Added + - Connection instrumentation events #110 + +## [1.3.0] - 2018-08-06 +### Fixed + - Skip unsubscribed topic assignment #86 + - Refresh metadata when producing to a topic without metadata #87 + - Discard messages with a lower offset than requested #100 + +### Added + - Add consumer auto commit policies #89 + - Notify user when setting heartbeat interval to same or higher than session timeout #91 + - Constantly refresh metadata based on `metadataMaxAge` #94 + - New instrumentation events #95 + - Expose loggers #97 #102 + - Add offset management operations to the admin client #101 + - Support to record batch compression #103 + - Handle missing username/password during authentication #104 + +## [1.2.0] - 2018-07-02 +### Fixed + - Make sure authentication handshake remains consistent event when `broker.connect` is called concurrently #81 + +### Added + - Add `producer.sendBatch` to produce to multiple topics at once #82 + - Experimental support to native Kafka 0.11 (`allowExperimentalV011`) #61 #68 #75 #76 #78 + - Enabled protocol `fetch` v3 + +## [1.1.0] - 2018-06-14 +### Added + - Support to SASL SCRAM (`scram-sha-256` and `scram-sha-512`) #72 + - Admin client with support to create topics #73 + +## [1.0.1] - 2018-05-18 +### Fixed + - Prevent crash when re-producing after metadata refresh #62 + +## [1.0.0] - 2018-05-14 +### Changed + - Updated readme + +## [0.8.1] - 2018-04-10 +### Fixed + - Throw unretriable error for incompatible message format #49 + - Producer can't reconnect after broker disconnection #48 + +## [0.8.0] - 2018-04-05 +### Removed + - Backwards compatibility with the old member assignment protocol (<= 0.6.x). __v0.7.x__ is the last version with support for both protocols (commit f965e91cf883bff74332e53a8c1d25c2af39e566) + +### Fixed + - Only retry failed brokers #38 + +### Changed + - Use selected assigner on consumer sync #35 + - Add validations to consumer subscribe and producer send #41 + +### Added + - Consumer pause and resume #41 + - Allow `partitionAssigners` to be configured + - Allow auto-resolve to be disabled for eachBatch #44 + +## [0.7.1] - 2018-01-22 +### Fixed + - Fix member assignment protocol #33 + +## [0.7.0] - 2018-01-19 +### Fixed + - Fix retry on error for message handlers #30 + +### Changed + - Use decoder offset for validating response length #21 + - Change log creator to improve interoperability #24 + - Add support to `KAFKAJS_LOG_LEVEL` #24 + - Improved assigner protocol #27 #29 + +### Added + - Add seek API to consumer #23 + - Add experimental describe group to consumer #31 + +## [0.6.8] - 2017-12-27 +### Fixed + - Only use seed brokers to bootstrap consumers (introduced a broker pool abstraction) #19 + - Don't include the size of the size field when determining if the buffer has enough bytes to read the full response #20 + +## [0.6.7] - 2017-12-18 +### Fixed + - Don't throw KafkaJSOffsetOutOfRange on every protocol error + +## [0.6.6] - 2017-12-15 +### Fixed + - Fix index out of range errors when decoding partial messages #11 + - Don't rely on seed broker for finding group coordinator #13 + +## [0.6.5] - 2017-12-14 +### Fixed + - Fix partial message decode #10 + - Throw not implemented error for Snappy and LZ4 compression #10 + - Don't crash when setting offsets to default #10 + +## [0.6.4] - 2017-12-13 +### Added + - Add stack trace to connection error logs + +### Changed + - Skip consumer callback error logs for kafkajs errors + +### Fixed + - Use the connection timeout callback for socket timeout + - Check if still connected before writing to the socket + +## [0.6.3] - 2017-12-11 +### Added + - Add error logs for user errors in `eachMessage` and `eachBatch` + +### Fixed + - Recover from rebalance in progress when starting the consumer + +## [0.6.2] - 2017-12-11 +### Added + - Add `logger: "kafkajs"` to log lines + +## [0.6.1] - 2017-12-08 +### Added + - Add latest Kafka error codes + +### Fixed + - Add fallback error for unsupported error codes + +## [0.6.0] - 2017-12-07 +### Added + - Expose consumer group `isRunning` to `eachBatch` + - Add `offsetLag` to consumer batch + +## [0.5.0] - 2017-12-04 +### Added + - Add ability to subscribe to events (heartbeats and offsets commit) #4 + +## [0.4.1] - 2017-11-30 +### Changed + - Add heartbeat after each message + - Update default heartbeat interval to a better value + +### Fixed + - Accept all consumer properties from create consumer + +## [0.4.0] - 2017-11-23 +### Added + - Commit previously resolved offsets when `eachBatch` throws an error + - Commit previously resolved offsets when `eachMessage` throws an error + - Consumer group recover from offset out of range + +## [0.3.2] - 2017-11-23 +### Fixed + - Stop consuming messages when the consumer group is not running + +## [0.3.1] - 2017-11-20 +### Fixed + - NPM bundle (add .npmignore) + - Fix package.json reference to main file + +## [0.3.0] - 2017-11-20 +### Added + - Add cluster method to fetch offsets for a list of topics + - Add offset resolution to the consumer group + +### Changed + - Rename protocol Offsets to ListOffsets + - Update cluster fetch topics offset to allow different configurations per topic + - Create different instances of the cluster for producers and consumers + +### Fixed + - Fix the use of timestamp in the ListOffsets protocol + - Fix create topic script + - Prevent unnecessary reconnections on cluster connect + +## [0.2.0] - 2017-11-13 +### Added + - Expose consumer groups + - Message and MessageSet V0 and V1 decoder + - Protocol fetch V0, V1, V2 and V3 + - Protocol find coordinator V0 + - Protocol join group V0 + - Protocol sync group V0 + - Protocol leave group V0 + - Protocol offsets V0 + - Protocol offset commit V0, V1 and V2 + - Protocol offset fetch V1 and V2 + - Protocol heartbeat V0 + - Support to compressed message sets in protocol fetch + - Add find coordinator group to cluster + - Set timestamp for compressed messages + - Travis integration + - Eslint and prettier + +### Changed + - Throw an error when cluster findBroker doesn't find a broker + - Move `KafkaProtocolError` to errors file + - Convert encode, decode and parse to async functions + - Update producer to use async compressor + - Update fetch to use async decompressor + - Create a separate error class for SASL errors + - Accepts a list of brokers instead of host and port + - Move retry logic out of connection and create namespaces for the logger + - Retry when refresh metadata throws leader not available + +### Fixed + - Fix broker maxWaitTime default value + - Take OS differences when asserting gzip results + - Clear the connection timeout when the connection ends or fails due to an error + +## [0.1.2] - 2017-10-17 +### Added + - Expose if the cluster is connected + +### Fixed + - Reconnect the cluster if it is not connected when producing messages + - Throw error if metadata is still not loaded when finding the topic partition + - Refresh metadata in case it is not loaded + - Make connection throw a retriable error when not connected + +## [0.1.1] - 2017-10-16 +### Changed + - Only retry retriable errors + - Propagate clientId and connectionTimeout to Cluster + +### Fixed + - Typo when loading the SASL Handshake protocol + +## [0.1.0] - 2017-10-15 +### Added + - Producer compatible with Kafka 0.10.x + - GZIP compression + - Plain, SSL and SASL_SSL implementations + - Published on Github diff --git a/node_modules/kafkajs/CONTRIBUTING.md b/node_modules/kafkajs/CONTRIBUTING.md new file mode 100644 index 0000000..9185e38 --- /dev/null +++ b/node_modules/kafkajs/CONTRIBUTING.md @@ -0,0 +1,40 @@ +# Contributing to KafkaJS + +Thank you for considering contributing to KafkaJS! + +Reading and following these guidelines will help us make the contribution process easy and effective for everyone involved. It also communicates that you agree to respect the time of the developers managing and developing these open source projects. + +## Getting Started + +Contributions are made to this repo via Issues and Pull Requests (PRs). A few general guidelines that cover both: + +- Search for existing issues and PRs before creating your own. +- The maintainers of KafkaJS are people that volunteer their time. We try to address issues and PRs in a timely manner, but cannot make any guarantees. Please don't @mention individual maintainers to try to get their attention. + +### Issues + +Issues should be used to report problems with the library, request a new feature, or to discuss potential changes before a PR is created. Please follow the issue template that's provided when you first create an issue in order to collect all the necessary information. + +Issues are not a support channel. Please use [StackOverflow](https://stackoverflow.com/questions/tagged/kafkajs), [Slack](https://join.slack.com/t/kafkajs/shared_invite/zt-1ezd5395v-SOpTqYoYfRCyPKOkUggK0A) or other online resources instead. [Limited support from a maintainer](https://github.com/sponsors/Nevon?frequency=one-time&sponsor=Nevon) is available to sponsors. + +If you find an issue that addresses the problem you're having, please add your own reproduction information to the existing issue rather than creating a new one. Adding a [reaction](https://github.blog/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) can also help be indicating to our maintainers that a particular problem is affecting more than just the reporter. + +### Pull Requests + +PRs are welcome and can be a quick way to get your fix or improvement out. If you've never contributed before, see [the contribution guidelines on our website](https://kafka.js.org/docs/contribution-guide) for practical information on how to get started. + +In general, PRs should: + +- Only fix/add the functionality in question **OR** address wide-spread whitespace/style issues, not both. +- Add tests for fixed or changed functionality. +- Address a single concern. +- Update the [Typescript type definitions](./types) if your change introduces any new or affects existing interfaces. +- Include documentation if it changes the functionality of the library. Our [documentation](https://kafka.js.org/docs/getting-started) is in the [`/docs`](./docs/) folder of the repo. + +If your PR introduces a change in functionality or adds new functionality, always open an issue first to discuss your proposal before implementing it. This is especially crucial for breaking changes, which will almost always be rejected unless discussed first. For bug fixes this is not required, but still recommended. + +Once a PR is merged and the master build is successful, a pre-release version of KafkaJS will be published to NPM in the [beta channel](https://www.npmjs.com/package/kafkajs/v/beta), which you can use until a there has been a stable release made containing your change. + +## Getting Help + +Join our [Slack community](https://join.slack.com/t/kafkajs/shared_invite/zt-1ezd5395v-SOpTqYoYfRCyPKOkUggK0A) if you have questions about the contribution process or otherwise want to get in touch. diff --git a/node_modules/kafkajs/LICENSE b/node_modules/kafkajs/LICENSE new file mode 100644 index 0000000..448bb65 --- /dev/null +++ b/node_modules/kafkajs/LICENSE @@ -0,0 +1,24 @@ +The MIT License + +Copyright (c) 2018 Túlio Ornelas (ornelas.tulio@gmail.com) + +Permission is hereby granted, free of charge, +to any person obtaining a copy of this software and +associated documentation files (the "Software"), to +deal in the Software without restriction, including +without limitation the rights to use, copy, modify, +merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom +the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR +ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/node_modules/kafkajs/README.md b/node_modules/kafkajs/README.md new file mode 100644 index 0000000..b828506 --- /dev/null +++ b/node_modules/kafkajs/README.md @@ -0,0 +1,178 @@ +[![npm version](https://img.shields.io/npm/v/kafkajs?color=%2344cc11&label=stable)](https://www.npmjs.com/package/kafkajs) [![npm pre-release version](https://img.shields.io/npm/v/kafkajs/beta?label=pre-release)](https://www.npmjs.com/package/kafkajs) [![Build Status](https://dev.azure.com/tulios/kafkajs/_apis/build/status/tulios.kafkajs?branchName=master)](https://dev.azure.com/tulios/kafkajs/_build/latest?definitionId=2&branchName=master) [![Slack Channel](https://join.slack.com/t/kafkajs/shared_invite/zt-1ezd5395v-SOpTqYoYfRCyPKOkUggK0Abadge.svg)](https://join.slack.com/t/kafkajs/shared_invite/zt-1ezd5395v-SOpTqYoYfRCyPKOkUggK0A) +
+

+ + Logo + + +

KafkaJS

+ +

+ A modern Apache Kafka® client for Node.js +
+ Get Started » +
+
+ Read the Docs + · + Report Bug + · + Request Feature +

+

+ +## Table of Contents + +- [About the project](#about) + - [Sponsors](#sponsorship) + - [Features](#features) + - [Getting Started](#getting-started) + - [Usage](#usage) +- [Contributing](#contributing) + - [Help Wanted](#help-wanted) + - [Contact](#contact) +- [License](#license) + - [Acknowledgements](#acknowledgements) + +## About the Project + +KafkaJS is a modern [Apache Kafka](https://kafka.apache.org/) client for Node.js. It is compatible with Kafka 0.10+ and offers native support for 0.11 features. + +KAFKA is a registered trademark of The Apache Software Foundation and has been licensed for use by KafkaJS. KafkaJS has no affiliation with and is not endorsed by The Apache Software Foundation. + +## Sponsors ❤️ + + + +*To become a sponsor, [reach out in our Slack community](https://join.slack.com/t/kafkajs/shared_invite/zt-1ezd5395v-SOpTqYoYfRCyPKOkUggK0A) to get in touch with one of the maintainers. Also consider becoming a Github Sponsor by following any of the links under "[Sponsor this project](https://github.com/tulios/kafkajs#sponsors)" in the sidebar.* + +### Features + +* Producer +* Consumer groups with pause, resume, and seek +* Transactional support for producers and consumers +* Message headers +* GZIP compression + * Snappy, LZ4 and ZSTD compression through pluggable codecs +* Plain, SSL and SASL_SSL implementations +* Support for SCRAM-SHA-256 and SCRAM-SHA-512 +* Support for AWS IAM authentication +* Admin client + +### Getting Started + +```sh +npm install kafkajs +# yarn add kafkajs +``` + +#### Usage +```javascript +const { Kafka } = require('kafkajs') + +const kafka = new Kafka({ + clientId: 'my-app', + brokers: ['kafka1:9092', 'kafka2:9092'] +}) + +const producer = kafka.producer() +const consumer = kafka.consumer({ groupId: 'test-group' }) + +const run = async () => { + // Producing + await producer.connect() + await producer.send({ + topic: 'test-topic', + messages: [ + { value: 'Hello KafkaJS user!' }, + ], + }) + + // Consuming + await consumer.connect() + await consumer.subscribe({ topic: 'test-topic', fromBeginning: true }) + + await consumer.run({ + eachMessage: async ({ topic, partition, message }) => { + console.log({ + partition, + offset: message.offset, + value: message.value.toString(), + }) + }, + }) +} + +run().catch(console.error) +``` + +Learn more about using [KafkaJS on the official site!](https://kafka.js.org) + +- [Getting Started](https://kafka.js.org/docs/getting-started) +- [A Brief Intro to Kafka](https://kafka.js.org/docs/introduction) +- [Configuring KafkaJS](https://kafka.js.org/docs/configuration) +- [Example Producer](https://kafka.js.org/docs/producer-example) +- [Example Consumer](https://kafka.js.org/docs/consumer-example) + +> _Read something on the website that didn't work with the latest stable version?_ +[Check the pre-release versions](https://kafka.js.org/docs/pre-releases) - the website is updated on every merge to master. + +## Contributing + +KafkaJS is an open-source project where development takes place in the open on GitHub. Although the project is maintained by a small group of dedicated volunteers, we are grateful to the community for bug fixes, feature development and other contributions. + +See [Developing KafkaJS](https://kafka.js.org/docs/contribution-guide) for information on how to run and develop KafkaJS. + +### Help wanted 🤝 + +We welcome contributions to KafkaJS, but we also want to see a thriving third-party ecosystem. If you would like to create an open-source project that builds on top of KafkaJS, [please get in touch](https://join.slack.com/t/kafkajs/shared_invite/zt-1ezd5395v-SOpTqYoYfRCyPKOkUggK0A) and we'd be happy to provide feedback and support. + +Here are some projects that we would like to build, but haven't yet been able to prioritize: + +* [Dead Letter Queue](https://eng.uber.com/reliable-reprocessing/) - Automatically reprocess messages +* ✅ [Schema Registry](https://www.confluent.io/confluent-schema-registry/) - **[Now available!](https://www.npmjs.com/package/@kafkajs/confluent-schema-registry)** thanks to [@erikengervall](https://github.com/erikengervall) +* [Metrics](https://prometheus.io/) - Integrate with the [instrumentation events](https://kafka.js.org/docs/instrumentation-events) to expose commonly used metrics + +### Contact 💬 + +[Join our Slack community](https://join.slack.com/t/kafkajs/shared_invite/zt-1ezd5395v-SOpTqYoYfRCyPKOkUggK0A) + +## License + +See [LICENSE](https://github.com/tulios/kafkajs/blob/master/LICENSE) for more details. + +### Acknowledgements + +* Thanks to [Sebastian Norde](https://github.com/sebastiannorde) for the V1 logo ❤️ +* Thanks to [Tracy (Tan Yun)](https://medium.com/@tanyuntracy) for the V2 logo ❤️ + +Apache Kafka and Kafka are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. KafkaJS has no affiliation with the Apache Software Foundation. diff --git a/node_modules/kafkajs/index.js b/node_modules/kafkajs/index.js new file mode 100644 index 0000000..3fd3cb1 --- /dev/null +++ b/node_modules/kafkajs/index.js @@ -0,0 +1,30 @@ +const Kafka = require('./src') +const PartitionAssigners = require('./src/consumer/assigners') +const AssignerProtocol = require('./src/consumer/assignerProtocol') +const Partitioners = require('./src/producer/partitioners') +const Compression = require('./src/protocol/message/compression') +const ConfigResourceTypes = require('./src/protocol/configResourceTypes') +const ConfigSource = require('./src/protocol/configSource') +const AclResourceTypes = require('./src/protocol/aclResourceTypes') +const AclOperationTypes = require('./src/protocol/aclOperationTypes') +const AclPermissionTypes = require('./src/protocol/aclPermissionTypes') +const ResourcePatternTypes = require('./src/protocol/resourcePatternTypes') +const { isRebalancing, isKafkaJSError, ...errors } = require('./src/errors') +const { LEVELS } = require('./src/loggers') + +module.exports = { + Kafka, + PartitionAssigners, + AssignerProtocol, + Partitioners, + logLevel: LEVELS, + CompressionTypes: Compression.Types, + CompressionCodecs: Compression.Codecs, + ConfigResourceTypes, + AclResourceTypes, + AclOperationTypes, + AclPermissionTypes, + ResourcePatternTypes, + ConfigSource, + ...errors, +} diff --git a/node_modules/kafkajs/package.json b/node_modules/kafkajs/package.json new file mode 100644 index 0000000..c8f58c3 --- /dev/null +++ b/node_modules/kafkajs/package.json @@ -0,0 +1,84 @@ +{ + "name": "kafkajs", + "version": "2.2.4", + "description": "A modern Apache Kafka client for node.js", + "author": "Tulio Ornelas ", + "main": "index.js", + "types": "types/index.d.ts", + "license": "MIT", + "keywords": [ + "kafka", + "sasl", + "scram" + ], + "engines": { + "node": ">=14.0.0" + }, + "repository": { + "type": "git", + "url": "https://github.com/tulios/kafkajs.git" + }, + "bugs": { + "url": "https://github.com/tulios/kafkajs/issues" + }, + "homepage": "https://kafka.js.org", + "scripts": { + "jest": "export KAFKA_VERSION=${KAFKA_VERSION:='2.4'} && NODE_ENV=test echo \"KAFKA_VERSION: ${KAFKA_VERSION}\" && KAFKAJS_DEBUG_PROTOCOL_BUFFERS=1 jest", + "test:local": "yarn jest --detectOpenHandles", + "test:debug": "NODE_ENV=test KAFKAJS_DEBUG_PROTOCOL_BUFFERS=1 node --inspect-brk $(yarn bin 2>/dev/null)/jest --detectOpenHandles --runInBand --watch", + "test:local:watch": "yarn test:local --watch", + "test": "yarn lint && JEST_JUNIT_OUTPUT_NAME=test-report.xml ./scripts/testWithKafka.sh 'yarn jest --ci --maxWorkers=4 --no-watchman --forceExit'", + "lint": "find . -path ./node_modules -prune -o -path ./coverage -prune -o -path ./website -prune -o -name '*.js' -print0 | xargs -0 eslint", + "format": "find . -path ./node_modules -prune -o -path ./coverage -prune -o -path ./website -prune -o -name '*.js' -print0 | xargs -0 prettier --write", + "precommit": "lint-staged", + "test:group:broker": "yarn jest --forceExit --testPathPattern 'src/broker/.*'", + "test:group:admin": "yarn jest --forceExit --testPathPattern 'src/admin/.*'", + "test:group:producer": "yarn jest --forceExit --testPathPattern 'src/producer/.*'", + "test:group:consumer": "yarn jest --forceExit --testPathPattern 'src/consumer/.*.spec.js'", + "test:group:others": "yarn jest --forceExit --testPathPattern 'src/(?!(broker|admin|producer|consumer)/).*'", + "test:group:oauthbearer": "OAUTHBEARER_ENABLED=1 yarn jest --forceExit src/producer/index.spec.js src/broker/__tests__/connect.spec.js src/consumer/__tests__/connection.spec.js src/broker/__tests__/disconnect.spec.js src/admin/__tests__/connection.spec.js src/broker/__tests__/reauthenticate.spec.js", + "test:group:broker:ci": "JEST_JUNIT_OUTPUT_NAME=test-report.xml ./scripts/testWithKafka.sh \"yarn test:group:broker --ci --maxWorkers=4 --no-watchman\"", + "test:group:admin:ci": "JEST_JUNIT_OUTPUT_NAME=test-report.xml ./scripts/testWithKafka.sh \"yarn test:group:admin --ci --maxWorkers=4 --no-watchman\"", + "test:group:producer:ci": "JEST_JUNIT_OUTPUT_NAME=test-report.xml ./scripts/testWithKafka.sh \"yarn test:group:producer --ci --maxWorkers=4 --no-watchman\"", + "test:group:consumer:ci": "JEST_JUNIT_OUTPUT_NAME=test-report.xml ./scripts/testWithKafka.sh \"yarn test:group:consumer --ci --maxWorkers=4 --no-watchman\"", + "test:group:others:ci": "JEST_JUNIT_OUTPUT_NAME=test-report.xml ./scripts/testWithKafka.sh \"yarn test:group:others --ci --maxWorkers=4 --no-watchman\"", + "test:group:oauthbearer:ci": "JEST_JUNIT_OUTPUT_NAME=test-report.xml COMPOSE_FILE='docker-compose.2_4_oauthbearer.yml' ./scripts/testWithKafka.sh \"yarn test:group:oauthbearer --ci --maxWorkers=4 --no-watchman\"", + "test:types": "tsc -p types/" + }, + "devDependencies": { + "@types/jest": "^27.4.0", + "@types/node": "^12.0.8", + "@typescript-eslint/typescript-estree": "^1.10.2", + "eslint": "^6.8.0", + "eslint-config-prettier": "^6.0.0", + "eslint-config-standard": "^13.0.1", + "eslint-plugin-import": "^2.18.2", + "eslint-plugin-jest": "^26.1.0", + "eslint-plugin-node": "^11.0.0", + "eslint-plugin-prettier": "^3.1.0", + "eslint-plugin-promise": "^4.2.1", + "eslint-plugin-standard": "^4.0.0", + "execa": "^2.0.3", + "glob": "^7.1.4", + "husky": "^3.0.1", + "ip": "^1.1.5", + "jest": "^25.1.0", + "jest-circus": "^25.1.0", + "jest-extended": "^0.11.2", + "jest-junit": "^10.0.0", + "jsonwebtoken": "^9.0.0", + "lint-staged": "^9.2.0", + "mockdate": "^2.0.5", + "prettier": "^1.18.2", + "semver": "^6.2.0", + "typescript": "^3.8.3", + "uuid": "^3.3.2" + }, + "dependencies": {}, + "lint-staged": { + "*.js": [ + "prettier --write", + "git add" + ] + } +} diff --git a/node_modules/kafkajs/src/admin/index.js b/node_modules/kafkajs/src/admin/index.js new file mode 100644 index 0000000..8e98691 --- /dev/null +++ b/node_modules/kafkajs/src/admin/index.js @@ -0,0 +1,1606 @@ +const createRetry = require('../retry') +const waitFor = require('../utils/waitFor') +const groupBy = require('../utils/groupBy') +const createConsumer = require('../consumer') +const InstrumentationEventEmitter = require('../instrumentation/emitter') +const { events, wrap: wrapEvent, unwrap: unwrapEvent } = require('./instrumentationEvents') +const { LEVELS } = require('../loggers') +const { + KafkaJSNonRetriableError, + KafkaJSDeleteGroupsError, + KafkaJSBrokerNotFound, + KafkaJSDeleteTopicRecordsError, + KafkaJSAggregateError, +} = require('../errors') +const { staleMetadata } = require('../protocol/error') +const CONFIG_RESOURCE_TYPES = require('../protocol/configResourceTypes') +const ACL_RESOURCE_TYPES = require('../protocol/aclResourceTypes') +const ACL_OPERATION_TYPES = require('../protocol/aclOperationTypes') +const ACL_PERMISSION_TYPES = require('../protocol/aclPermissionTypes') +const RESOURCE_PATTERN_TYPES = require('../protocol/resourcePatternTypes') +const { EARLIEST_OFFSET, LATEST_OFFSET } = require('../constants') + +const { CONNECT, DISCONNECT } = events + +const NO_CONTROLLER_ID = -1 + +const { values, keys, entries } = Object +const eventNames = values(events) +const eventKeys = keys(events) + .map(key => `admin.events.${key}`) + .join(', ') + +const retryOnLeaderNotAvailable = (fn, opts = {}) => { + const callback = async () => { + try { + return await fn() + } catch (e) { + if (e.type !== 'LEADER_NOT_AVAILABLE') { + throw e + } + return false + } + } + + return waitFor(callback, opts) +} + +const isConsumerGroupRunning = description => ['Empty', 'Dead'].includes(description.state) +const findTopicPartitions = async (cluster, topic) => { + await cluster.addTargetTopic(topic) + await cluster.refreshMetadataIfNecessary() + + return cluster + .findTopicPartitionMetadata(topic) + .map(({ partitionId }) => partitionId) + .sort() +} +const indexByPartition = array => + array.reduce( + (obj, { partition, ...props }) => Object.assign(obj, { [partition]: { ...props } }), + {} + ) + +/** + * + * @param {Object} params + * @param {import("../../types").Logger} params.logger + * @param {InstrumentationEventEmitter} [params.instrumentationEmitter] + * @param {import('../../types').RetryOptions} params.retry + * @param {import("../../types").Cluster} params.cluster + * + * @returns {import("../../types").Admin} + */ +module.exports = ({ + logger: rootLogger, + instrumentationEmitter: rootInstrumentationEmitter, + retry, + cluster, +}) => { + const logger = rootLogger.namespace('Admin') + const instrumentationEmitter = rootInstrumentationEmitter || new InstrumentationEventEmitter() + + /** + * @returns {Promise} + */ + const connect = async () => { + await cluster.connect() + instrumentationEmitter.emit(CONNECT) + } + + /** + * @return {Promise} + */ + const disconnect = async () => { + await cluster.disconnect() + instrumentationEmitter.emit(DISCONNECT) + } + + /** + * @return {Promise} + */ + const listTopics = async () => { + const { topicMetadata } = await cluster.metadata() + const topics = topicMetadata.map(t => t.topic) + return topics + } + + /** + * @param {Object} request + * @param {array} request.topics + * @param {boolean} [request.validateOnly=false] + * @param {number} [request.timeout=5000] + * @param {boolean} [request.waitForLeaders=true] + * @return {Promise} + */ + const createTopics = async ({ topics, validateOnly, timeout, waitForLeaders = true }) => { + if (!topics || !Array.isArray(topics)) { + throw new KafkaJSNonRetriableError(`Invalid topics array ${topics}`) + } + + if (topics.filter(({ topic }) => typeof topic !== 'string').length > 0) { + throw new KafkaJSNonRetriableError( + 'Invalid topics array, the topic names have to be a valid string' + ) + } + + const topicNames = new Set(topics.map(({ topic }) => topic)) + if (topicNames.size < topics.length) { + throw new KafkaJSNonRetriableError( + 'Invalid topics array, it cannot have multiple entries for the same topic' + ) + } + + for (const { topic, configEntries } of topics) { + if (configEntries == null) { + continue + } + + if (!Array.isArray(configEntries)) { + throw new KafkaJSNonRetriableError( + `Invalid configEntries for topic "${topic}", must be an array` + ) + } + + configEntries.forEach((entry, index) => { + if (typeof entry !== 'object' || entry == null) { + throw new KafkaJSNonRetriableError( + `Invalid configEntries for topic "${topic}". Entry ${index} must be an object` + ) + } + + for (const requiredProperty of ['name', 'value']) { + if ( + !Object.prototype.hasOwnProperty.call(entry, requiredProperty) || + typeof entry[requiredProperty] !== 'string' + ) { + throw new KafkaJSNonRetriableError( + `Invalid configEntries for topic "${topic}". Entry ${index} must have a valid "${requiredProperty}" property` + ) + } + } + }) + } + + const retrier = createRetry(retry) + + return retrier(async (bail, retryCount, retryTime) => { + try { + await cluster.refreshMetadata() + const broker = await cluster.findControllerBroker() + await broker.createTopics({ topics, validateOnly, timeout }) + + if (waitForLeaders) { + const topicNamesArray = Array.from(topicNames.values()) + await retryOnLeaderNotAvailable(async () => await broker.metadata(topicNamesArray), { + delay: 100, + maxWait: timeout, + timeoutMessage: 'Timed out while waiting for topic leaders', + }) + } + + return true + } catch (e) { + if (e.type === 'NOT_CONTROLLER') { + logger.warn('Could not create topics', { error: e.message, retryCount, retryTime }) + throw e + } + + if (e instanceof KafkaJSAggregateError) { + if (e.errors.every(error => error.type === 'TOPIC_ALREADY_EXISTS')) { + return false + } + } + + bail(e) + } + }) + } + /** + * @param {array} topicPartitions + * @param {boolean} [validateOnly=false] + * @param {number} [timeout=5000] + * @return {Promise} + */ + const createPartitions = async ({ topicPartitions, validateOnly, timeout }) => { + if (!topicPartitions || !Array.isArray(topicPartitions)) { + throw new KafkaJSNonRetriableError(`Invalid topic partitions array ${topicPartitions}`) + } + if (topicPartitions.length === 0) { + throw new KafkaJSNonRetriableError(`Empty topic partitions array`) + } + + if (topicPartitions.filter(({ topic }) => typeof topic !== 'string').length > 0) { + throw new KafkaJSNonRetriableError( + 'Invalid topic partitions array, the topic names have to be a valid string' + ) + } + + const topicNames = new Set(topicPartitions.map(({ topic }) => topic)) + if (topicNames.size < topicPartitions.length) { + throw new KafkaJSNonRetriableError( + 'Invalid topic partitions array, it cannot have multiple entries for the same topic' + ) + } + + const retrier = createRetry(retry) + + return retrier(async (bail, retryCount, retryTime) => { + try { + await cluster.refreshMetadata() + const broker = await cluster.findControllerBroker() + await broker.createPartitions({ topicPartitions, validateOnly, timeout }) + } catch (e) { + if (e.type === 'NOT_CONTROLLER') { + logger.warn('Could not create topics', { error: e.message, retryCount, retryTime }) + throw e + } + + bail(e) + } + }) + } + + /** + * @param {string[]} topics + * @param {number} [timeout=5000] + * @return {Promise} + */ + const deleteTopics = async ({ topics, timeout }) => { + if (!topics || !Array.isArray(topics)) { + throw new KafkaJSNonRetriableError(`Invalid topics array ${topics}`) + } + + if (topics.filter(topic => typeof topic !== 'string').length > 0) { + throw new KafkaJSNonRetriableError('Invalid topics array, the names must be a valid string') + } + + const retrier = createRetry(retry) + + return retrier(async (bail, retryCount, retryTime) => { + try { + await cluster.refreshMetadata() + const broker = await cluster.findControllerBroker() + await broker.deleteTopics({ topics, timeout }) + + // Remove deleted topics + for (const topic of topics) { + cluster.targetTopics.delete(topic) + } + + await cluster.refreshMetadata() + } catch (e) { + if (['NOT_CONTROLLER', 'UNKNOWN_TOPIC_OR_PARTITION'].includes(e.type)) { + logger.warn('Could not delete topics', { error: e.message, retryCount, retryTime }) + throw e + } + + if (e.type === 'REQUEST_TIMED_OUT') { + logger.error( + 'Could not delete topics, check if "delete.topic.enable" is set to "true" (the default value is "false") or increase the timeout', + { + error: e.message, + retryCount, + retryTime, + } + ) + } + + bail(e) + } + }) + } + + /** + * @param {string} topic + */ + + const fetchTopicOffsets = async topic => { + if (!topic || typeof topic !== 'string') { + throw new KafkaJSNonRetriableError(`Invalid topic ${topic}`) + } + + const retrier = createRetry(retry) + + return retrier(async (bail, retryCount, retryTime) => { + try { + await cluster.addTargetTopic(topic) + await cluster.refreshMetadataIfNecessary() + + const metadata = cluster.findTopicPartitionMetadata(topic) + const high = await cluster.fetchTopicsOffset([ + { + topic, + fromBeginning: false, + partitions: metadata.map(p => ({ partition: p.partitionId })), + }, + ]) + + const low = await cluster.fetchTopicsOffset([ + { + topic, + fromBeginning: true, + partitions: metadata.map(p => ({ partition: p.partitionId })), + }, + ]) + + const { partitions: highPartitions } = high.pop() + const { partitions: lowPartitions } = low.pop() + return highPartitions.map(({ partition, offset }) => ({ + partition, + offset, + high: offset, + low: lowPartitions.find(({ partition: lowPartition }) => lowPartition === partition) + .offset, + })) + } catch (e) { + if (e.type === 'UNKNOWN_TOPIC_OR_PARTITION') { + await cluster.refreshMetadata() + throw e + } + + bail(e) + } + }) + } + + /** + * @param {string} topic + * @param {number} [timestamp] + */ + + const fetchTopicOffsetsByTimestamp = async (topic, timestamp) => { + if (!topic || typeof topic !== 'string') { + throw new KafkaJSNonRetriableError(`Invalid topic ${topic}`) + } + + const retrier = createRetry(retry) + + return retrier(async (bail, retryCount, retryTime) => { + try { + await cluster.addTargetTopic(topic) + await cluster.refreshMetadataIfNecessary() + + const metadata = cluster.findTopicPartitionMetadata(topic) + const partitions = metadata.map(p => ({ partition: p.partitionId })) + + const high = await cluster.fetchTopicsOffset([ + { + topic, + fromBeginning: false, + partitions, + }, + ]) + const { partitions: highPartitions } = high.pop() + + const offsets = await cluster.fetchTopicsOffset([ + { + topic, + fromTimestamp: timestamp, + partitions, + }, + ]) + const { partitions: lowPartitions } = offsets.pop() + + return lowPartitions.map(({ partition, offset }) => ({ + partition, + offset: + parseInt(offset, 10) >= 0 + ? offset + : highPartitions.find(({ partition: highPartition }) => highPartition === partition) + .offset, + })) + } catch (e) { + if (e.type === 'UNKNOWN_TOPIC_OR_PARTITION') { + await cluster.refreshMetadata() + throw e + } + + bail(e) + } + }) + } + + /** + * Fetch offsets for a topic or multiple topics + * + * Note: set either topic or topics but not both. + * + * @param {string} groupId + * @param {string[]} topics - list of topics to fetch offsets for, defaults to `[]` which fetches all topics for `groupId`. + * @param {boolean} [resolveOffsets=false] + * @return {Promise} + */ + const fetchOffsets = async ({ groupId, topics, resolveOffsets = false }) => { + if (!groupId) { + throw new KafkaJSNonRetriableError(`Invalid groupId ${groupId}`) + } + + if (!topics) { + topics = [] + } + + if (!Array.isArray(topics)) { + throw new KafkaJSNonRetriableError('Expected topics array to be set') + } + + const coordinator = await cluster.findGroupCoordinator({ groupId }) + const topicsToFetch = await Promise.all( + topics.map(async topic => { + const partitions = await findTopicPartitions(cluster, topic) + const partitionsToFetch = partitions.map(partition => ({ partition })) + return { topic, partitions: partitionsToFetch } + }) + ) + let { responses: consumerOffsets } = await coordinator.offsetFetch({ + groupId, + topics: topicsToFetch, + }) + + if (resolveOffsets) { + consumerOffsets = await Promise.all( + consumerOffsets.map(async ({ topic, partitions }) => { + const indexedOffsets = indexByPartition(await fetchTopicOffsets(topic)) + const recalculatedPartitions = partitions.map(({ offset, partition, ...props }) => { + let resolvedOffset = offset + if (Number(offset) === EARLIEST_OFFSET) { + resolvedOffset = indexedOffsets[partition].low + } + if (Number(offset) === LATEST_OFFSET) { + resolvedOffset = indexedOffsets[partition].high + } + return { + partition, + offset: resolvedOffset, + ...props, + } + }) + + await setOffsets({ groupId, topic, partitions: recalculatedPartitions }) + + return { + topic, + partitions: recalculatedPartitions, + } + }) + ) + } + + return consumerOffsets.map(({ topic, partitions }) => { + const completePartitions = partitions.map(({ partition, offset, metadata }) => ({ + partition, + offset, + metadata: metadata || null, + })) + + return { topic, partitions: completePartitions } + }) + } + + /** + * @param {string} groupId + * @param {string} topic + * @param {boolean} [earliest=false] + * @return {Promise} + */ + const resetOffsets = async ({ groupId, topic, earliest = false }) => { + if (!groupId) { + throw new KafkaJSNonRetriableError(`Invalid groupId ${groupId}`) + } + + if (!topic) { + throw new KafkaJSNonRetriableError(`Invalid topic ${topic}`) + } + + const partitions = await findTopicPartitions(cluster, topic) + const partitionsToSeek = partitions.map(partition => ({ + partition, + offset: cluster.defaultOffset({ fromBeginning: earliest }), + })) + + return setOffsets({ groupId, topic, partitions: partitionsToSeek }) + } + + /** + * @param {string} groupId + * @param {string} topic + * @param {Array} partitions + * @return {Promise} + * + * @typedef {Object} SeekEntry + * @property {number} partition + * @property {string} offset + */ + const setOffsets = async ({ groupId, topic, partitions }) => { + if (!groupId) { + throw new KafkaJSNonRetriableError(`Invalid groupId ${groupId}`) + } + + if (!topic) { + throw new KafkaJSNonRetriableError(`Invalid topic ${topic}`) + } + + if (!partitions || partitions.length === 0) { + throw new KafkaJSNonRetriableError(`Invalid partitions`) + } + + const consumer = createConsumer({ + logger: rootLogger.namespace('Admin', LEVELS.NOTHING), + cluster, + groupId, + }) + + await consumer.subscribe({ topic, fromBeginning: true }) + const description = await consumer.describeGroup() + + if (!isConsumerGroupRunning(description)) { + throw new KafkaJSNonRetriableError( + `The consumer group must have no running instances, current state: ${description.state}` + ) + } + + return new Promise((resolve, reject) => { + consumer.on(consumer.events.FETCH, async () => + consumer + .stop() + .then(resolve) + .catch(reject) + ) + + consumer + .run({ + eachBatchAutoResolve: false, + eachBatch: async () => true, + }) + .catch(reject) + + // This consumer doesn't need to consume any data + consumer.pause([{ topic }]) + + for (const seekData of partitions) { + consumer.seek({ topic, ...seekData }) + } + }) + } + + const isBrokerConfig = type => + [CONFIG_RESOURCE_TYPES.BROKER, CONFIG_RESOURCE_TYPES.BROKER_LOGGER].includes(type) + + /** + * Broker configs can only be returned by the target broker + * + * @see + * https://github.com/apache/kafka/blob/821c1ac6641845aeca96a43bc2b946ecec5cba4f/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java#L3783 + * https://github.com/apache/kafka/blob/821c1ac6641845aeca96a43bc2b946ecec5cba4f/clients/src/main/java/org/apache/kafka/clients/admin/KafkaAdminClient.java#L2027 + * + * @param {Broker} defaultBroker. Broker used in case the configuration is not a broker config + */ + const groupResourcesByBroker = ({ resources, defaultBroker }) => + groupBy(resources, async ({ type, name: nodeId }) => { + return isBrokerConfig(type) + ? await cluster.findBroker({ nodeId: String(nodeId) }) + : defaultBroker + }) + + /** + * @param {Array} resources + * @param {boolean} [includeSynonyms=false] + * @return {Promise} + * + * @typedef {Object} ResourceConfigQuery + * @property {ConfigResourceType} type + * @property {string} name + * @property {Array} [configNames=[]] + */ + const describeConfigs = async ({ resources, includeSynonyms }) => { + if (!resources || !Array.isArray(resources)) { + throw new KafkaJSNonRetriableError(`Invalid resources array ${resources}`) + } + + if (resources.length === 0) { + throw new KafkaJSNonRetriableError('Resources array cannot be empty') + } + + const validResourceTypes = Object.values(CONFIG_RESOURCE_TYPES) + const invalidType = resources.find(r => !validResourceTypes.includes(r.type)) + + if (invalidType) { + throw new KafkaJSNonRetriableError( + `Invalid resource type ${invalidType.type}: ${JSON.stringify(invalidType)}` + ) + } + + const invalidName = resources.find(r => !r.name || typeof r.name !== 'string') + + if (invalidName) { + throw new KafkaJSNonRetriableError( + `Invalid resource name ${invalidName.name}: ${JSON.stringify(invalidName)}` + ) + } + + const invalidConfigs = resources.find( + r => !Array.isArray(r.configNames) && r.configNames != null + ) + + if (invalidConfigs) { + const { configNames } = invalidConfigs + throw new KafkaJSNonRetriableError( + `Invalid resource configNames ${configNames}: ${JSON.stringify(invalidConfigs)}` + ) + } + + const retrier = createRetry(retry) + + return retrier(async (bail, retryCount, retryTime) => { + try { + await cluster.refreshMetadata() + const controller = await cluster.findControllerBroker() + const resourcerByBroker = await groupResourcesByBroker({ + resources, + defaultBroker: controller, + }) + + const describeConfigsAction = async broker => { + const targetBroker = broker || controller + return targetBroker.describeConfigs({ + resources: resourcerByBroker.get(targetBroker), + includeSynonyms, + }) + } + + const brokers = Array.from(resourcerByBroker.keys()) + const responses = await Promise.all(brokers.map(describeConfigsAction)) + const responseResources = responses.reduce( + (result, { resources }) => [...result, ...resources], + [] + ) + + return { resources: responseResources } + } catch (e) { + if (e.type === 'NOT_CONTROLLER') { + logger.warn('Could not describe configs', { error: e.message, retryCount, retryTime }) + throw e + } + + bail(e) + } + }) + } + + /** + * @param {Array} resources + * @param {boolean} [validateOnly=false] + * @return {Promise} + * + * @typedef {Object} ResourceConfig + * @property {ConfigResourceType} type + * @property {string} name + * @property {Array} configEntries + * + * @typedef {Object} ResourceConfigEntry + * @property {string} name + * @property {string} value + */ + const alterConfigs = async ({ resources, validateOnly }) => { + if (!resources || !Array.isArray(resources)) { + throw new KafkaJSNonRetriableError(`Invalid resources array ${resources}`) + } + + if (resources.length === 0) { + throw new KafkaJSNonRetriableError('Resources array cannot be empty') + } + + const validResourceTypes = Object.values(CONFIG_RESOURCE_TYPES) + const invalidType = resources.find(r => !validResourceTypes.includes(r.type)) + + if (invalidType) { + throw new KafkaJSNonRetriableError( + `Invalid resource type ${invalidType.type}: ${JSON.stringify(invalidType)}` + ) + } + + const invalidName = resources.find(r => !r.name || typeof r.name !== 'string') + + if (invalidName) { + throw new KafkaJSNonRetriableError( + `Invalid resource name ${invalidName.name}: ${JSON.stringify(invalidName)}` + ) + } + + const invalidConfigs = resources.find(r => !Array.isArray(r.configEntries)) + + if (invalidConfigs) { + const { configEntries } = invalidConfigs + throw new KafkaJSNonRetriableError( + `Invalid resource configEntries ${configEntries}: ${JSON.stringify(invalidConfigs)}` + ) + } + + const invalidConfigValue = resources.find(r => + r.configEntries.some(e => typeof e.name !== 'string' || typeof e.value !== 'string') + ) + + if (invalidConfigValue) { + throw new KafkaJSNonRetriableError( + `Invalid resource config value: ${JSON.stringify(invalidConfigValue)}` + ) + } + + const retrier = createRetry(retry) + + return retrier(async (bail, retryCount, retryTime) => { + try { + await cluster.refreshMetadata() + const controller = await cluster.findControllerBroker() + const resourcerByBroker = await groupResourcesByBroker({ + resources, + defaultBroker: controller, + }) + + const alterConfigsAction = async broker => { + const targetBroker = broker || controller + return targetBroker.alterConfigs({ + resources: resourcerByBroker.get(targetBroker), + validateOnly: !!validateOnly, + }) + } + + const brokers = Array.from(resourcerByBroker.keys()) + const responses = await Promise.all(brokers.map(alterConfigsAction)) + const responseResources = responses.reduce( + (result, { resources }) => [...result, ...resources], + [] + ) + + return { resources: responseResources } + } catch (e) { + if (e.type === 'NOT_CONTROLLER') { + logger.warn('Could not alter configs', { error: e.message, retryCount, retryTime }) + throw e + } + + bail(e) + } + }) + } + + /** + * Fetch metadata for provided topics. + * + * If no topics are provided fetch metadata for all topics. + * @see https://kafka.apache.org/protocol#The_Messages_Metadata + * + * @param {Object} [options] + * @param {string[]} [options.topics] + * @return {Promise} + * + * @typedef {Object} TopicsMetadata + * @property {Array} topics + * + * @typedef {Object} TopicMetadata + * @property {String} name + * @property {Array} partitions + * + * @typedef {Object} PartitionMetadata + * @property {number} partitionErrorCode Response error code + * @property {number} partitionId Topic partition id + * @property {number} leader The id of the broker acting as leader for this partition. + * @property {Array} replicas The set of all nodes that host this partition. + * @property {Array} isr The set of nodes that are in sync with the leader for this partition. + */ + const fetchTopicMetadata = async ({ topics = [] } = {}) => { + if (topics) { + topics.forEach(topic => { + if (!topic || typeof topic !== 'string') { + throw new KafkaJSNonRetriableError(`Invalid topic ${topic}`) + } + }) + } + + const metadata = await cluster.metadata({ topics }) + + return { + topics: metadata.topicMetadata.map(topicMetadata => ({ + name: topicMetadata.topic, + partitions: topicMetadata.partitionMetadata, + })), + } + } + + /** + * Describe cluster + * + * @return {Promise} + * + * @typedef {Object} ClusterMetadata + * @property {Array} brokers + * @property {Number} controller Current controller id. Returns null if unknown. + * @property {String} clusterId + * + * @typedef {Object} Broker + * @property {Number} nodeId + * @property {String} host + * @property {Number} port + */ + const describeCluster = async () => { + const { brokers: nodes, clusterId, controllerId } = await cluster.metadata({ topics: [] }) + const brokers = nodes.map(({ nodeId, host, port }) => ({ + nodeId, + host, + port, + })) + const controller = + controllerId == null || controllerId === NO_CONTROLLER_ID ? null : controllerId + + return { + brokers, + controller, + clusterId, + } + } + + /** + * List groups in a broker + * + * @return {Promise} + * + * @typedef {Object} ListGroups + * @property {Array} groups + * + * @typedef {Object} ListGroup + * @property {string} groupId + * @property {string} protocolType + */ + const listGroups = async () => { + await cluster.refreshMetadata() + let groups = [] + for (var nodeId in cluster.brokerPool.brokers) { + const broker = await cluster.findBroker({ nodeId }) + const response = await broker.listGroups() + groups = groups.concat(response.groups) + } + + return { groups } + } + + /** + * Describe groups by group ids + * @param {Array} groupIds + * + * @typedef {Object} GroupDescriptions + * @property {Array} groups + * + * @return {Promise} + */ + const describeGroups = async groupIds => { + const coordinatorsForGroup = await Promise.all( + groupIds.map(async groupId => { + const coordinator = await cluster.findGroupCoordinator({ groupId }) + return { + coordinator, + groupId, + } + }) + ) + + const groupsByCoordinator = Object.values( + coordinatorsForGroup.reduce((coordinators, { coordinator, groupId }) => { + const group = coordinators[coordinator.nodeId] + + if (group) { + coordinators[coordinator.nodeId] = { + ...group, + groupIds: [...group.groupIds, groupId], + } + } else { + coordinators[coordinator.nodeId] = { coordinator, groupIds: [groupId] } + } + return coordinators + }, {}) + ) + + const responses = await Promise.all( + groupsByCoordinator.map(async ({ coordinator, groupIds }) => { + const retrier = createRetry(retry) + const { groups } = await retrier(() => coordinator.describeGroups({ groupIds })) + return groups + }) + ) + + const groups = [].concat.apply([], responses) + + return { groups } + } + + /** + * Delete groups in a broker + * + * @param {string[]} [groupIds] + * @return {Promise} + * + * @typedef {Array} DeleteGroups + * @property {string} groupId + * @property {number} errorCode + */ + const deleteGroups = async groupIds => { + if (!groupIds || !Array.isArray(groupIds)) { + throw new KafkaJSNonRetriableError(`Invalid groupIds array ${groupIds}`) + } + + const invalidGroupId = groupIds.some(g => typeof g !== 'string') + + if (invalidGroupId) { + throw new KafkaJSNonRetriableError(`Invalid groupId name: ${JSON.stringify(invalidGroupId)}`) + } + + const retrier = createRetry(retry) + + let results = [] + + let clonedGroupIds = groupIds.slice() + + return retrier(async (bail, retryCount, retryTime) => { + try { + if (clonedGroupIds.length === 0) return [] + + await cluster.refreshMetadata() + + const brokersPerGroups = {} + const brokersPerNode = {} + for (const groupId of clonedGroupIds) { + const broker = await cluster.findGroupCoordinator({ groupId }) + if (brokersPerGroups[broker.nodeId] === undefined) brokersPerGroups[broker.nodeId] = [] + brokersPerGroups[broker.nodeId].push(groupId) + brokersPerNode[broker.nodeId] = broker + } + + const res = await Promise.all( + Object.keys(brokersPerNode).map( + async nodeId => await brokersPerNode[nodeId].deleteGroups(brokersPerGroups[nodeId]) + ) + ) + + const errors = res + .flatMap(({ results }) => + results.map(({ groupId, errorCode, error }) => { + return { groupId, errorCode, error } + }) + ) + .filter(({ errorCode }) => errorCode !== 0) + + clonedGroupIds = errors.map(({ groupId }) => groupId) + + if (errors.length > 0) throw new KafkaJSDeleteGroupsError('Error in DeleteGroups', errors) + + results = res.flatMap(({ results }) => results) + + return results + } catch (e) { + if (e.type === 'NOT_CONTROLLER' || e.type === 'COORDINATOR_NOT_AVAILABLE') { + logger.warn('Could not delete groups', { error: e.message, retryCount, retryTime }) + throw e + } + + bail(e) + } + }) + } + + /** + * Delete topic records up to the selected partition offsets + * + * @param {string} topic + * @param {Array} partitions + * @return {Promise} + * + * @typedef {Object} SeekEntry + * @property {number} partition + * @property {string} offset + */ + const deleteTopicRecords = async ({ topic, partitions }) => { + if (!topic || typeof topic !== 'string') { + throw new KafkaJSNonRetriableError(`Invalid topic "${topic}"`) + } + + if (!partitions || partitions.length === 0) { + throw new KafkaJSNonRetriableError(`Invalid partitions`) + } + + const partitionsByBroker = cluster.findLeaderForPartitions( + topic, + partitions.map(p => p.partition) + ) + + const partitionsFound = values(partitionsByBroker).flat() + const topicOffsets = await fetchTopicOffsets(topic) + + const leaderNotFoundErrors = [] + partitions.forEach(({ partition, offset }) => { + // throw if no leader found for partition + if (!partitionsFound.includes(partition)) { + leaderNotFoundErrors.push({ + partition, + offset, + error: new KafkaJSBrokerNotFound('Could not find the leader for the partition', { + retriable: false, + }), + }) + return + } + const { low } = topicOffsets.find(p => p.partition === partition) || { + high: undefined, + low: undefined, + } + // warn in case of offset below low watermark + if (parseInt(offset) < parseInt(low) && parseInt(offset) !== -1) { + logger.warn( + 'The requested offset is before the earliest offset maintained on the partition - no records will be deleted from this partition', + { + topic, + partition, + offset, + } + ) + } + }) + + if (leaderNotFoundErrors.length > 0) { + throw new KafkaJSDeleteTopicRecordsError({ topic, partitions: leaderNotFoundErrors }) + } + + const seekEntriesByBroker = entries(partitionsByBroker).reduce( + (obj, [nodeId, nodePartitions]) => { + obj[nodeId] = { + topic, + partitions: partitions.filter(p => nodePartitions.includes(p.partition)), + } + return obj + }, + {} + ) + + const retrier = createRetry(retry) + return retrier(async bail => { + try { + const partitionErrors = [] + + const brokerRequests = entries(seekEntriesByBroker).map( + ([nodeId, { topic, partitions }]) => async () => { + const broker = await cluster.findBroker({ nodeId }) + await broker.deleteRecords({ topics: [{ topic, partitions }] }) + // remove successful entry so it's ignored on retry + delete seekEntriesByBroker[nodeId] + } + ) + + await Promise.all( + brokerRequests.map(request => + request().catch(e => { + if (e.name === 'KafkaJSDeleteTopicRecordsError') { + e.partitions.forEach(({ partition, offset, error }) => { + partitionErrors.push({ + partition, + offset, + error, + }) + }) + } else { + // then it's an unknown error, not from the broker response + throw e + } + }) + ) + ) + + if (partitionErrors.length > 0) { + throw new KafkaJSDeleteTopicRecordsError({ + topic, + partitions: partitionErrors, + }) + } + } catch (e) { + if ( + e.retriable && + e.partitions.some( + ({ error }) => staleMetadata(error) || error.name === 'KafkaJSMetadataNotLoaded' + ) + ) { + await cluster.refreshMetadata() + } + throw e + } + }) + } + + /** + * @param {Array} acl + * @return {Promise} + * + * @typedef {Object} ACLEntry + */ + const createAcls = async ({ acl }) => { + if (!acl || !Array.isArray(acl)) { + throw new KafkaJSNonRetriableError(`Invalid ACL array ${acl}`) + } + if (acl.length === 0) { + throw new KafkaJSNonRetriableError('Empty ACL array') + } + + // Validate principal + if (acl.some(({ principal }) => typeof principal !== 'string')) { + throw new KafkaJSNonRetriableError( + 'Invalid ACL array, the principals have to be a valid string' + ) + } + + // Validate host + if (acl.some(({ host }) => typeof host !== 'string')) { + throw new KafkaJSNonRetriableError('Invalid ACL array, the hosts have to be a valid string') + } + + // Validate resourceName + if (acl.some(({ resourceName }) => typeof resourceName !== 'string')) { + throw new KafkaJSNonRetriableError( + 'Invalid ACL array, the resourceNames have to be a valid string' + ) + } + + let invalidType + // Validate operation + const validOperationTypes = Object.values(ACL_OPERATION_TYPES) + invalidType = acl.find(i => !validOperationTypes.includes(i.operation)) + + if (invalidType) { + throw new KafkaJSNonRetriableError( + `Invalid operation type ${invalidType.operation}: ${JSON.stringify(invalidType)}` + ) + } + + // Validate resourcePatternTypes + const validResourcePatternTypes = Object.values(RESOURCE_PATTERN_TYPES) + invalidType = acl.find(i => !validResourcePatternTypes.includes(i.resourcePatternType)) + + if (invalidType) { + throw new KafkaJSNonRetriableError( + `Invalid resource pattern type ${invalidType.resourcePatternType}: ${JSON.stringify( + invalidType + )}` + ) + } + + // Validate permissionTypes + const validPermissionTypes = Object.values(ACL_PERMISSION_TYPES) + invalidType = acl.find(i => !validPermissionTypes.includes(i.permissionType)) + + if (invalidType) { + throw new KafkaJSNonRetriableError( + `Invalid permission type ${invalidType.permissionType}: ${JSON.stringify(invalidType)}` + ) + } + + // Validate resourceTypes + const validResourceTypes = Object.values(ACL_RESOURCE_TYPES) + invalidType = acl.find(i => !validResourceTypes.includes(i.resourceType)) + + if (invalidType) { + throw new KafkaJSNonRetriableError( + `Invalid resource type ${invalidType.resourceType}: ${JSON.stringify(invalidType)}` + ) + } + + const retrier = createRetry(retry) + + return retrier(async (bail, retryCount, retryTime) => { + try { + await cluster.refreshMetadata() + const broker = await cluster.findControllerBroker() + await broker.createAcls({ acl }) + + return true + } catch (e) { + if (e.type === 'NOT_CONTROLLER') { + logger.warn('Could not create ACL', { error: e.message, retryCount, retryTime }) + throw e + } + + bail(e) + } + }) + } + + /** + * @param {ACLResourceTypes} resourceType The type of resource + * @param {string} resourceName The name of the resource + * @param {ACLResourcePatternTypes} resourcePatternType The resource pattern type filter + * @param {string} principal The principal name + * @param {string} host The hostname + * @param {ACLOperationTypes} operation The type of operation + * @param {ACLPermissionTypes} permissionType The type of permission + * @return {Promise} + * + * @typedef {number} ACLResourceTypes + * @typedef {number} ACLResourcePatternTypes + * @typedef {number} ACLOperationTypes + * @typedef {number} ACLPermissionTypes + */ + const describeAcls = async ({ + resourceType, + resourceName, + resourcePatternType, + principal, + host, + operation, + permissionType, + }) => { + // Validate principal + if (typeof principal !== 'string' && typeof principal !== 'undefined') { + throw new KafkaJSNonRetriableError( + 'Invalid principal, the principal have to be a valid string' + ) + } + + // Validate host + if (typeof host !== 'string' && typeof host !== 'undefined') { + throw new KafkaJSNonRetriableError('Invalid host, the host have to be a valid string') + } + + // Validate resourceName + if (typeof resourceName !== 'string' && typeof resourceName !== 'undefined') { + throw new KafkaJSNonRetriableError( + 'Invalid resourceName, the resourceName have to be a valid string' + ) + } + + // Validate operation + const validOperationTypes = Object.values(ACL_OPERATION_TYPES) + if (!validOperationTypes.includes(operation)) { + throw new KafkaJSNonRetriableError(`Invalid operation type ${operation}`) + } + + // Validate resourcePatternType + const validResourcePatternTypes = Object.values(RESOURCE_PATTERN_TYPES) + if (!validResourcePatternTypes.includes(resourcePatternType)) { + throw new KafkaJSNonRetriableError( + `Invalid resource pattern filter type ${resourcePatternType}` + ) + } + + // Validate permissionType + const validPermissionTypes = Object.values(ACL_PERMISSION_TYPES) + if (!validPermissionTypes.includes(permissionType)) { + throw new KafkaJSNonRetriableError(`Invalid permission type ${permissionType}`) + } + + // Validate resourceType + const validResourceTypes = Object.values(ACL_RESOURCE_TYPES) + if (!validResourceTypes.includes(resourceType)) { + throw new KafkaJSNonRetriableError(`Invalid resource type ${resourceType}`) + } + + const retrier = createRetry(retry) + + return retrier(async (bail, retryCount, retryTime) => { + try { + await cluster.refreshMetadata() + const broker = await cluster.findControllerBroker() + const { resources } = await broker.describeAcls({ + resourceType, + resourceName, + resourcePatternType, + principal, + host, + operation, + permissionType, + }) + return { resources } + } catch (e) { + if (e.type === 'NOT_CONTROLLER') { + logger.warn('Could not describe ACL', { error: e.message, retryCount, retryTime }) + throw e + } + + bail(e) + } + }) + } + + /** + * @param {Array} filters + * @return {Promise} + * + * @typedef {Object} ACLFilter + */ + const deleteAcls = async ({ filters }) => { + if (!filters || !Array.isArray(filters)) { + throw new KafkaJSNonRetriableError(`Invalid ACL Filter array ${filters}`) + } + + if (filters.length === 0) { + throw new KafkaJSNonRetriableError('Empty ACL Filter array') + } + + // Validate principal + if ( + filters.some( + ({ principal }) => typeof principal !== 'string' && typeof principal !== 'undefined' + ) + ) { + throw new KafkaJSNonRetriableError( + 'Invalid ACL Filter array, the principals have to be a valid string' + ) + } + + // Validate host + if (filters.some(({ host }) => typeof host !== 'string' && typeof host !== 'undefined')) { + throw new KafkaJSNonRetriableError( + 'Invalid ACL Filter array, the hosts have to be a valid string' + ) + } + + // Validate resourceName + if ( + filters.some( + ({ resourceName }) => + typeof resourceName !== 'string' && typeof resourceName !== 'undefined' + ) + ) { + throw new KafkaJSNonRetriableError( + 'Invalid ACL Filter array, the resourceNames have to be a valid string' + ) + } + + let invalidType + // Validate operation + const validOperationTypes = Object.values(ACL_OPERATION_TYPES) + invalidType = filters.find(i => !validOperationTypes.includes(i.operation)) + + if (invalidType) { + throw new KafkaJSNonRetriableError( + `Invalid operation type ${invalidType.operation}: ${JSON.stringify(invalidType)}` + ) + } + + // Validate resourcePatternTypes + const validResourcePatternTypes = Object.values(RESOURCE_PATTERN_TYPES) + invalidType = filters.find(i => !validResourcePatternTypes.includes(i.resourcePatternType)) + + if (invalidType) { + throw new KafkaJSNonRetriableError( + `Invalid resource pattern type ${invalidType.resourcePatternType}: ${JSON.stringify( + invalidType + )}` + ) + } + + // Validate permissionTypes + const validPermissionTypes = Object.values(ACL_PERMISSION_TYPES) + invalidType = filters.find(i => !validPermissionTypes.includes(i.permissionType)) + + if (invalidType) { + throw new KafkaJSNonRetriableError( + `Invalid permission type ${invalidType.permissionType}: ${JSON.stringify(invalidType)}` + ) + } + + // Validate resourceTypes + const validResourceTypes = Object.values(ACL_RESOURCE_TYPES) + invalidType = filters.find(i => !validResourceTypes.includes(i.resourceType)) + + if (invalidType) { + throw new KafkaJSNonRetriableError( + `Invalid resource type ${invalidType.resourceType}: ${JSON.stringify(invalidType)}` + ) + } + + const retrier = createRetry(retry) + + return retrier(async (bail, retryCount, retryTime) => { + try { + await cluster.refreshMetadata() + const broker = await cluster.findControllerBroker() + const { filterResponses } = await broker.deleteAcls({ filters }) + return { filterResponses } + } catch (e) { + if (e.type === 'NOT_CONTROLLER') { + logger.warn('Could not delete ACL', { error: e.message, retryCount, retryTime }) + throw e + } + + bail(e) + } + }) + } + + /** + * Alter the replicas partitions are assigned to for a topic + * @param {Object} request + * @param {import("../../types").IPartitionReassignment[]} request.topics topics and the paritions to be reassigned + * @param {number} [request.timeout] + * @returns {Promise} + */ + const alterPartitionReassignments = async ({ topics, timeout }) => { + if (!topics || !Array.isArray(topics)) { + throw new KafkaJSNonRetriableError(`Invalid topics array ${topics}`) + } + + if (topics.filter(({ topic }) => typeof topic !== 'string').length > 0) { + throw new KafkaJSNonRetriableError( + 'Invalid topics array, the topic names have to be a valid string' + ) + } + + const topicNames = new Set(topics.map(({ topic }) => topic)) + if (topicNames.size < topics.length) { + throw new KafkaJSNonRetriableError( + 'Invalid topics array, it cannot have multiple entries for the same topic' + ) + } + + for (const { topic, partitionAssignment } of topics) { + if (!partitionAssignment || !Array.isArray(partitionAssignment)) { + throw new KafkaJSNonRetriableError( + `Invalid partitions array: ${partitionAssignment} for topic: ${topic}` + ) + } + + for (const { partition, replicas } of partitionAssignment) { + if ( + partition === null || + partition === undefined || + typeof partition !== 'number' || + partition < 0 + ) { + throw new KafkaJSNonRetriableError( + `Invalid partitions index: ${partition} for topic: ${topic}` + ) + } + + if (!replicas || !Array.isArray(replicas)) { + throw new KafkaJSNonRetriableError( + `Invalid replica assignment: ${replicas} for topic: ${topic} on partition: ${partition}` + ) + } + + if (replicas.filter(replica => typeof replica !== 'number' || replica < 0).length >= 1) { + throw new KafkaJSNonRetriableError( + `Invalid replica assignment: ${replicas} for topic: ${topic} on partition: ${partition}. Replicas must be a non negative number` + ) + } + } + } + + const retrier = createRetry(retry) + + return retrier(async (bail, retryCount, retryTime) => { + try { + await cluster.refreshMetadata() + const broker = await cluster.findControllerBroker() + await broker.alterPartitionReassignments({ topics, timeout }) + } catch (e) { + if (e.type === 'NOT_CONTROLLER') { + logger.warn('Could not reassign partitions', { error: e.message, retryCount, retryTime }) + throw e + } + + bail(e) + } + }) + } + + /** + * List the partition reassignments in progress. + * If a partition is not going through a reassignment, its AddingReplicas and RemovingReplicas fields will simply be empty. + * If a partition doesn't exist, no response will be returned for it. + * @param {Object} request + * @param {import("../../types").TopicPartitions[]} request.topics topics and the paritions to be returned, if this is null will return all the topics. + * @param {number} [request.timeout] + * @returns {Promise} + */ + const listPartitionReassignments = async ({ topics = null, timeout }) => { + if (topics) { + if (!Array.isArray(topics)) { + throw new KafkaJSNonRetriableError(`Invalid topics array ${topics}`) + } + + if (topics.filter(({ topic }) => typeof topic !== 'string').length > 0) { + throw new KafkaJSNonRetriableError( + 'Invalid topics array, the topic names have to be a valid string' + ) + } + + const topicNames = new Set(topics.map(({ topic }) => topic)) + if (topicNames.size < topics.length) { + throw new KafkaJSNonRetriableError( + 'Invalid topics array, it cannot have multiple entries for the same topic' + ) + } + + for (const { topic, partitions } of topics) { + if (!partitions || !Array.isArray(partitions)) { + throw new KafkaJSNonRetriableError( + `Invalid partition array: ${partitions} for topic: ${topic}` + ) + } + + if ( + partitions.filter(partition => typeof partition !== 'number' || partition < 0).length >= 1 + ) { + throw new KafkaJSNonRetriableError( + `Invalid partition array: ${partitions} for topic: ${topic}. The partition indices have to be a valid number greater than 0.` + ) + } + } + } + + const retrier = createRetry(retry) + + return retrier(async (bail, retryCount, retryTime) => { + try { + await cluster.refreshMetadata() + const broker = await cluster.findControllerBroker() + const response = await broker.listPartitionReassignments({ topics, timeout }) + + return { topics: response.topics } + } catch (e) { + if (e.type === 'NOT_CONTROLLER') { + logger.warn('Could not reassign partitions', { error: e.message, retryCount, retryTime }) + throw e + } + + bail(e) + } + }) + } + + /** @type {import("../../types").Admin["on"]} */ + const on = (eventName, listener) => { + if (!eventNames.includes(eventName)) { + throw new KafkaJSNonRetriableError(`Event name should be one of ${eventKeys}`) + } + + return instrumentationEmitter.addListener(unwrapEvent(eventName), event => { + event.type = wrapEvent(event.type) + Promise.resolve(listener(event)).catch(e => { + logger.error(`Failed to execute listener: ${e.message}`, { + eventName, + stack: e.stack, + }) + }) + }) + } + + /** + * @return {Object} logger + */ + const getLogger = () => logger + + return { + connect, + disconnect, + listTopics, + createTopics, + deleteTopics, + createPartitions, + fetchTopicMetadata, + describeCluster, + events, + fetchOffsets, + fetchTopicOffsets, + fetchTopicOffsetsByTimestamp, + setOffsets, + resetOffsets, + describeConfigs, + alterConfigs, + on, + logger: getLogger, + listGroups, + describeGroups, + deleteGroups, + describeAcls, + deleteAcls, + createAcls, + deleteTopicRecords, + alterPartitionReassignments, + listPartitionReassignments, + } +} diff --git a/node_modules/kafkajs/src/admin/instrumentationEvents.js b/node_modules/kafkajs/src/admin/instrumentationEvents.js new file mode 100644 index 0000000..cfc095c --- /dev/null +++ b/node_modules/kafkajs/src/admin/instrumentationEvents.js @@ -0,0 +1,28 @@ +const swapObject = require('../utils/swapObject') +const networkEvents = require('../network/instrumentationEvents') +const InstrumentationEventType = require('../instrumentation/eventType') +const adminType = InstrumentationEventType('admin') + +const events = { + CONNECT: adminType('connect'), + DISCONNECT: adminType('disconnect'), + REQUEST: adminType(networkEvents.NETWORK_REQUEST), + REQUEST_TIMEOUT: adminType(networkEvents.NETWORK_REQUEST_TIMEOUT), + REQUEST_QUEUE_SIZE: adminType(networkEvents.NETWORK_REQUEST_QUEUE_SIZE), +} + +const wrappedEvents = { + [events.REQUEST]: networkEvents.NETWORK_REQUEST, + [events.REQUEST_TIMEOUT]: networkEvents.NETWORK_REQUEST_TIMEOUT, + [events.REQUEST_QUEUE_SIZE]: networkEvents.NETWORK_REQUEST_QUEUE_SIZE, +} + +const reversedWrappedEvents = swapObject(wrappedEvents) +const unwrap = eventName => wrappedEvents[eventName] || eventName +const wrap = eventName => reversedWrappedEvents[eventName] || eventName + +module.exports = { + events, + wrap, + unwrap, +} diff --git a/node_modules/kafkajs/src/broker/index.js b/node_modules/kafkajs/src/broker/index.js new file mode 100644 index 0000000..bb86cc5 --- /dev/null +++ b/node_modules/kafkajs/src/broker/index.js @@ -0,0 +1,913 @@ +const Lock = require('../utils/lock') +const { Types: Compression } = require('../protocol/message/compression') +const { requests, lookup } = require('../protocol/requests') +const { KafkaJSNonRetriableError } = require('../errors') +const apiKeys = require('../protocol/requests/apiKeys') +const shuffle = require('../utils/shuffle') + +const PRIVATE = { + SEND_REQUEST: Symbol('private:Broker:sendRequest'), +} + +/** @type {import("../protocol/requests").Lookup} */ +const notInitializedLookup = () => { + throw new Error('Broker not connected') +} + +/** + * Each node in a Kafka cluster is called broker. This class contains + * the high-level operations a node can perform. + * + * @type {import("../../types").Broker} + */ +module.exports = class Broker { + /** + * @param {Object} options + * @param {import("../network/connectionPool")} options.connectionPool + * @param {import("../../types").Logger} options.logger + * @param {number} [options.nodeId] + * @param {import("../../types").ApiVersions} [options.versions=null] The object with all available versions and APIs + * supported by this cluster. The output of broker#apiVersions + * @param {number} [options.authenticationTimeout=10000] + * @param {boolean} [options.allowAutoTopicCreation=true] If this and the broker config 'auto.create.topics.enable' + * are true, topics that don't exist will be created when + * fetching metadata. + */ + constructor({ + connectionPool, + logger, + nodeId = null, + versions = null, + authenticationTimeout = 10000, + allowAutoTopicCreation = true, + }) { + this.connectionPool = connectionPool + this.nodeId = nodeId + this.rootLogger = logger + this.logger = logger.namespace('Broker') + this.versions = versions + this.authenticationTimeout = authenticationTimeout + this.allowAutoTopicCreation = allowAutoTopicCreation + + // The lock timeout has twice the connectionTimeout because the same timeout is used + // for the first apiVersions call + const lockTimeout = 2 * this.connectionPool.connectionTimeout + this.authenticationTimeout + this.brokerAddress = `${this.connectionPool.host}:${this.connectionPool.port}` + + this.lock = new Lock({ + timeout: lockTimeout, + description: `connect to broker ${this.brokerAddress}`, + }) + + this.lookupRequest = notInitializedLookup + } + + /** + * @public + * @returns {boolean} + */ + isConnected() { + return this.connectionPool.sasl + ? this.connectionPool.isConnected() && this.connectionPool.isAuthenticated() + : this.connectionPool.isConnected() + } + + /** + * @public + * @returns {Promise} + */ + async connect() { + await this.lock.acquire() + try { + if (this.isConnected()) { + return + } + + const connection = await this.connectionPool.getConnection() + + if (!this.versions) { + this.versions = await this.apiVersions() + } + this.connectionPool.setVersions(this.versions) + + this.lookupRequest = lookup(this.versions) + + if (connection.getSupportAuthenticationProtocol() === null) { + let supportAuthenticationProtocol = false + try { + this.lookupRequest(apiKeys.SaslAuthenticate, requests.SaslAuthenticate) + supportAuthenticationProtocol = true + } catch (_) { + supportAuthenticationProtocol = false + } + this.connectionPool.setSupportAuthenticationProtocol(supportAuthenticationProtocol) + + this.logger.debug(`Verified support for SaslAuthenticate`, { + broker: this.brokerAddress, + supportAuthenticationProtocol, + }) + } + + await connection.authenticate() + } finally { + await this.lock.release() + } + } + + /** + * @public + * @returns {Promise} + */ + async disconnect() { + await this.connectionPool.destroy() + } + + /** + * @public + * @returns {Promise} + */ + async apiVersions() { + let response + const availableVersions = requests.ApiVersions.versions + .map(Number) + .sort() + .reverse() + + // Find the best version implemented by the server + for (const candidateVersion of availableVersions) { + try { + const apiVersions = requests.ApiVersions.protocol({ version: candidateVersion }) + response = await this[PRIVATE.SEND_REQUEST]({ + ...apiVersions(), + requestTimeout: this.connectionPool.connectionTimeout, + }) + break + } catch (e) { + if (e.type !== 'UNSUPPORTED_VERSION') { + throw e + } + } + } + + if (!response) { + throw new KafkaJSNonRetriableError('API Versions not supported') + } + + return response.apiVersions.reduce( + (obj, version) => + Object.assign(obj, { + [version.apiKey]: { + minVersion: version.minVersion, + maxVersion: version.maxVersion, + }, + }), + {} + ) + } + + /** + * @public + * @type {import("../../types").Broker['metadata']} + * @param {string[]} [topics=[]] An array of topics to fetch metadata for. + * If no topics are specified fetch metadata for all topics + */ + async metadata(topics = []) { + const metadata = this.lookupRequest(apiKeys.Metadata, requests.Metadata) + const shuffledTopics = shuffle(topics) + return await this[PRIVATE.SEND_REQUEST]( + metadata({ topics: shuffledTopics, allowAutoTopicCreation: this.allowAutoTopicCreation }) + ) + } + + /** + * @public + * @param {Object} request + * @param {Array} request.topicData An array of messages per topic and per partition, example: + * [ + * { + * topic: 'test-topic-1', + * partitions: [ + * { + * partition: 0, + * firstSequence: 0, + * messages: [ + * { key: '1', value: 'A' }, + * { key: '2', value: 'B' }, + * ] + * }, + * { + * partition: 1, + * firstSequence: 0, + * messages: [ + * { key: '3', value: 'C' }, + * ] + * } + * ] + * }, + * { + * topic: 'test-topic-2', + * partitions: [ + * { + * partition: 4, + * firstSequence: 0, + * messages: [ + * { key: '32', value: 'E' }, + * ] + * }, + * ] + * }, + * ] + * @param {number} [request.acks=-1] Control the number of required acks. + * -1 = all replicas must acknowledge + * 0 = no acknowledgments + * 1 = only waits for the leader to acknowledge + * @param {number} [request.timeout=30000] The time to await a response in ms + * @param {string} [request.transactionalId=null] + * @param {number} [request.producerId=-1] Broker assigned producerId + * @param {number} [request.producerEpoch=0] Broker assigned producerEpoch + * @param {import("../../types").CompressionTypes} [request.compression=CompressionTypes.None] Compression codec + * @returns {Promise} + */ + async produce({ + topicData, + transactionalId, + producerId, + producerEpoch, + acks = -1, + timeout = 30000, + compression = Compression.None, + }) { + const produce = this.lookupRequest(apiKeys.Produce, requests.Produce) + return await this[PRIVATE.SEND_REQUEST]( + produce({ + acks, + timeout, + compression, + topicData, + transactionalId, + producerId, + producerEpoch, + }) + ) + } + + /** + * @public + * @param {Object} request + * @param {number} [request.replicaId=-1] Broker id of the follower. For normal consumers, use -1 + * @param {number} [request.isolationLevel=1] This setting controls the visibility of transactional records. Default READ_COMMITTED. + * @param {number} [request.maxWaitTime=5000] Maximum time in ms to wait for the response + * @param {number} [request.minBytes=1] Minimum bytes to accumulate in the response + * @param {number} [request.maxBytes=10485760] Maximum bytes to accumulate in the response. Note that this is + * not an absolute maximum, if the first message in the first non-empty + * partition of the fetch is larger than this value, the message will still + * be returned to ensure that progress can be made. Default 10MB. + * @param {Array} request.topics Topics to fetch + * [ + * { + * topic: 'topic-name', + * partitions: [ + * { + * partition: 0, + * fetchOffset: '4124', + * maxBytes: 2048 + * } + * ] + * } + * ] + * @param {string} [request.rackId=''] A rack identifier for this client. This can be any string value which indicates where this + * client is physically located. It corresponds with the broker config `broker.rack`. + * @returns {Promise} + */ + async fetch({ + replicaId, + isolationLevel, + maxWaitTime = 5000, + minBytes = 1, + maxBytes = 10485760, + topics, + rackId = '', + }) { + // TODO: validate topics not null/empty + const fetch = this.lookupRequest(apiKeys.Fetch, requests.Fetch) + + // Shuffle topic-partitions to ensure fair response allocation across partitions (KIP-74) + const flattenedTopicPartitions = topics.reduce((topicPartitions, { topic, partitions }) => { + partitions.forEach(partition => { + topicPartitions.push({ topic, partition }) + }) + return topicPartitions + }, []) + + const shuffledTopicPartitions = shuffle(flattenedTopicPartitions) + + // Consecutive partitions for the same topic can be combined into a single `topic` entry + const consolidatedTopicPartitions = shuffledTopicPartitions.reduce( + (topicPartitions, { topic, partition }) => { + const last = topicPartitions[topicPartitions.length - 1] + + if (last != null && last.topic === topic) { + topicPartitions[topicPartitions.length - 1].partitions.push(partition) + } else { + topicPartitions.push({ topic, partitions: [partition] }) + } + + return topicPartitions + }, + [] + ) + + return await this[PRIVATE.SEND_REQUEST]( + fetch({ + replicaId, + isolationLevel, + maxWaitTime, + minBytes, + maxBytes, + topics: consolidatedTopicPartitions, + rackId, + }) + ) + } + + /** + * @public + * @param {object} request + * @param {string} request.groupId The group id + * @param {number} request.groupGenerationId The generation of the group + * @param {string} request.memberId The member id assigned by the group coordinator + * @returns {Promise} + */ + async heartbeat({ groupId, groupGenerationId, memberId }) { + const heartbeat = this.lookupRequest(apiKeys.Heartbeat, requests.Heartbeat) + return await this[PRIVATE.SEND_REQUEST](heartbeat({ groupId, groupGenerationId, memberId })) + } + + /** + * @public + * @param {object} request + * @param {string} request.groupId The unique group id + * @param {import("../protocol/coordinatorTypes").CoordinatorType} request.coordinatorType The type of coordinator to find + * @returns {Promise} + */ + async findGroupCoordinator({ groupId, coordinatorType }) { + // TODO: validate groupId, mandatory + const findCoordinator = this.lookupRequest(apiKeys.GroupCoordinator, requests.GroupCoordinator) + return await this[PRIVATE.SEND_REQUEST](findCoordinator({ groupId, coordinatorType })) + } + + /** + * @public + * @param {object} request + * @param {string} request.groupId The unique group id + * @param {number} request.sessionTimeout The coordinator considers the consumer dead if it receives + * no heartbeat after this timeout in ms + * @param {number} request.rebalanceTimeout The maximum time that the coordinator will wait for each member + * to rejoin when rebalancing the group + * @param {string} [request.memberId=""] The assigned consumer id or an empty string for a new consumer + * @param {string} [request.protocolType="consumer"] Unique name for class of protocols implemented by group + * @param {Array} request.groupProtocols List of protocols that the member supports (assignment strategy) + * [{ name: 'AssignerName', metadata: '{"version": 1, "topics": []}' }] + * @returns {Promise} + */ + async joinGroup({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId = '', + protocolType = 'consumer', + groupProtocols, + }) { + const joinGroup = this.lookupRequest(apiKeys.JoinGroup, requests.JoinGroup) + const makeRequest = (assignedMemberId = memberId) => + this[PRIVATE.SEND_REQUEST]( + joinGroup({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId: assignedMemberId, + protocolType, + groupProtocols, + }) + ) + + try { + return await makeRequest() + } catch (error) { + if (error.name === 'KafkaJSMemberIdRequired') { + return makeRequest(error.memberId) + } + + throw error + } + } + + /** + * @public + * @param {object} request + * @param {string} request.groupId + * @param {string} request.memberId + * @returns {Promise} + */ + async leaveGroup({ groupId, memberId }) { + const leaveGroup = this.lookupRequest(apiKeys.LeaveGroup, requests.LeaveGroup) + return await this[PRIVATE.SEND_REQUEST](leaveGroup({ groupId, memberId })) + } + + /** + * @public + * @param {object} request + * @param {string} request.groupId + * @param {number} request.generationId + * @param {string} request.memberId + * @param {object} request.groupAssignment + * @returns {Promise} + */ + async syncGroup({ groupId, generationId, memberId, groupAssignment }) { + const syncGroup = this.lookupRequest(apiKeys.SyncGroup, requests.SyncGroup) + return await this[PRIVATE.SEND_REQUEST]( + syncGroup({ + groupId, + generationId, + memberId, + groupAssignment, + }) + ) + } + + /** + * @public + * @param {object} request + * @param {number} request.replicaId=-1 Broker id of the follower. For normal consumers, use -1 + * @param {number} request.isolationLevel=1 This setting controls the visibility of transactional records (default READ_COMMITTED, Kafka >0.11 only) + * @param {TopicPartitionOffset[]} request.topics e.g: + * + * @typedef {Object} TopicPartitionOffset + * @property {string} topic + * @property {PartitionOffset[]} partitions + * + * @typedef {Object} PartitionOffset + * @property {number} partition + * @property {number} [timestamp=-1] + * + * + * @returns {Promise} + */ + async listOffsets({ replicaId, isolationLevel, topics }) { + const listOffsets = this.lookupRequest(apiKeys.ListOffsets, requests.ListOffsets) + const result = await this[PRIVATE.SEND_REQUEST]( + listOffsets({ replicaId, isolationLevel, topics }) + ) + + // ListOffsets >= v1 will return a single `offset` rather than an array of `offsets` (ListOffsets V0). + // Normalize to just return `offset`. + for (const response of result.responses) { + response.partitions = response.partitions.map(({ offsets, ...partitionData }) => { + return offsets ? { ...partitionData, offset: offsets.pop() } : partitionData + }) + } + + return result + } + + /** + * @public + * @param {object} request + * @param {string} request.groupId + * @param {number} request.groupGenerationId + * @param {string} request.memberId + * @param {number} [request.retentionTime=-1] -1 signals to the broker that its default configuration + * should be used. + * @param {object} request.topics Topics to commit offsets, e.g: + * [ + * { + * topic: 'topic-name', + * partitions: [ + * { partition: 0, offset: '11' } + * ] + * } + * ] + * @returns {Promise} + */ + async offsetCommit({ groupId, groupGenerationId, memberId, retentionTime, topics }) { + const offsetCommit = this.lookupRequest(apiKeys.OffsetCommit, requests.OffsetCommit) + return await this[PRIVATE.SEND_REQUEST]( + offsetCommit({ + groupId, + groupGenerationId, + memberId, + retentionTime, + topics, + }) + ) + } + + /** + * @public + * @param {object} request + * @param {string} request.groupId + * @param {object} request.topics - If the topic array is null fetch offsets for all topics. e.g: + * [ + * { + * topic: 'topic-name', + * partitions: [ + * { partition: 0 } + * ] + * } + * ] + * @returns {Promise} + */ + async offsetFetch({ groupId, topics }) { + const offsetFetch = this.lookupRequest(apiKeys.OffsetFetch, requests.OffsetFetch) + return await this[PRIVATE.SEND_REQUEST](offsetFetch({ groupId, topics })) + } + + /** + * @public + * @param {object} request + * @param {Array} request.groupIds + * @returns {Promise} + */ + async describeGroups({ groupIds }) { + const describeGroups = this.lookupRequest(apiKeys.DescribeGroups, requests.DescribeGroups) + return await this[PRIVATE.SEND_REQUEST](describeGroups({ groupIds })) + } + + /** + * @public + * @param {object} request + * @param {Array} request.topics e.g: + * [ + * { + * topic: 'topic-name', + * numPartitions: 1, + * replicationFactor: 1 + * } + * ] + * @param {boolean} [request.validateOnly=false] If this is true, the request will be validated, but the topic + * won't be created + * @param {number} [request.timeout=5000] The time in ms to wait for a topic to be completely created + * on the controller node + * @returns {Promise} + */ + async createTopics({ topics, validateOnly = false, timeout = 5000 }) { + const createTopics = this.lookupRequest(apiKeys.CreateTopics, requests.CreateTopics) + return await this[PRIVATE.SEND_REQUEST](createTopics({ topics, validateOnly, timeout })) + } + + /** + * @public + * @param {object} request + * @param {Array} request.topicPartitions e.g: + * [ + * { + * topic: 'topic-name', + * count: 3, + * assignments: [] + * } + * ] + * @param {boolean} [request.validateOnly=false] If this is true, the request will be validated, but the topic + * won't be created + * @param {number} [request.timeout=5000] The time in ms to wait for a topic to be completely created + * on the controller node + * @returns {Promise} + */ + async createPartitions({ topicPartitions, validateOnly = false, timeout = 5000 }) { + const createPartitions = this.lookupRequest(apiKeys.CreatePartitions, requests.CreatePartitions) + return await this[PRIVATE.SEND_REQUEST]( + createPartitions({ topicPartitions, validateOnly, timeout }) + ) + } + + /** + * @public + * @param {object} request + * @param {string[]} request.topics An array of topics to be deleted + * @param {number} [request.timeout=5000] The time in ms to wait for a topic to be completely deleted on the + * controller node. + * @returns {Promise} + */ + async deleteTopics({ topics, timeout = 5000 }) { + const deleteTopics = this.lookupRequest(apiKeys.DeleteTopics, requests.DeleteTopics) + return await this[PRIVATE.SEND_REQUEST](deleteTopics({ topics, timeout })) + } + + /** + * @public + * @param {object} request + * @param {import("../../types").ResourceConfigQuery[]} request.resources + * [{ + * type: RESOURCE_TYPES.TOPIC, + * name: 'topic-name', + * configNames: ['compression.type', 'retention.ms'] + * }] + * @param {boolean} [request.includeSynonyms=false] + * @returns {Promise} + */ + async describeConfigs({ resources, includeSynonyms = false }) { + const describeConfigs = this.lookupRequest(apiKeys.DescribeConfigs, requests.DescribeConfigs) + return await this[PRIVATE.SEND_REQUEST](describeConfigs({ resources, includeSynonyms })) + } + + /** + * @public + * @param {object} request + * @param {import("../../types").IResourceConfig[]} request.resources + * [{ + * type: RESOURCE_TYPES.TOPIC, + * name: 'topic-name', + * configEntries: [ + * { + * name: 'cleanup.policy', + * value: 'compact' + * } + * ] + * }] + * @param {boolean} [request.validateOnly=false] + * @returns {Promise} + */ + async alterConfigs({ resources, validateOnly = false }) { + const alterConfigs = this.lookupRequest(apiKeys.AlterConfigs, requests.AlterConfigs) + return await this[PRIVATE.SEND_REQUEST](alterConfigs({ resources, validateOnly })) + } + + /** + * Send an `InitProducerId` request to fetch a PID and bump the producer epoch. + * + * Request should be made to the transaction coordinator. + * @public + * @param {object} request + * @param {number} request.transactionTimeout The time in ms to wait for before aborting idle transactions + * @param {number} [request.transactionalId] The transactional id or null if the producer is not transactional + * @returns {Promise} + */ + async initProducerId({ transactionalId, transactionTimeout }) { + const initProducerId = this.lookupRequest(apiKeys.InitProducerId, requests.InitProducerId) + return await this[PRIVATE.SEND_REQUEST](initProducerId({ transactionalId, transactionTimeout })) + } + + /** + * Send an `AddPartitionsToTxn` request to mark a TopicPartition as participating in the transaction. + * + * Request should be made to the transaction coordinator. + * @public + * @param {object} request + * @param {string} request.transactionalId The transactional id corresponding to the transaction. + * @param {number} request.producerId Current producer id in use by the transactional id. + * @param {number} request.producerEpoch Current epoch associated with the producer id. + * @param {object[]} request.topics e.g: + * [ + * { + * topic: 'topic-name', + * partitions: [ 0, 1] + * } + * ] + * @returns {Promise} + */ + async addPartitionsToTxn({ transactionalId, producerId, producerEpoch, topics }) { + const addPartitionsToTxn = this.lookupRequest( + apiKeys.AddPartitionsToTxn, + requests.AddPartitionsToTxn + ) + return await this[PRIVATE.SEND_REQUEST]( + addPartitionsToTxn({ transactionalId, producerId, producerEpoch, topics }) + ) + } + + /** + * Send an `AddOffsetsToTxn` request. + * + * Request should be made to the transaction coordinator. + * @public + * @param {object} request + * @param {string} request.transactionalId The transactional id corresponding to the transaction. + * @param {number} request.producerId Current producer id in use by the transactional id. + * @param {number} request.producerEpoch Current epoch associated with the producer id. + * @param {string} request.groupId The unique group identifier (for the consumer group) + * @returns {Promise} + */ + async addOffsetsToTxn({ transactionalId, producerId, producerEpoch, groupId }) { + const addOffsetsToTxn = this.lookupRequest(apiKeys.AddOffsetsToTxn, requests.AddOffsetsToTxn) + return await this[PRIVATE.SEND_REQUEST]( + addOffsetsToTxn({ transactionalId, producerId, producerEpoch, groupId }) + ) + } + + /** + * Send a `TxnOffsetCommit` request to persist the offsets in the `__consumer_offsets` topics. + * + * Request should be made to the consumer coordinator. + * @public + * @param {object} request + * @param {OffsetCommitTopic[]} request.topics + * @param {string} request.transactionalId The transactional id corresponding to the transaction. + * @param {string} request.groupId The unique group identifier (for the consumer group) + * @param {number} request.producerId Current producer id in use by the transactional id. + * @param {number} request.producerEpoch Current epoch associated with the producer id. + * @param {OffsetCommitTopic[]} request.topics + * + * @typedef {Object} OffsetCommitTopic + * @property {string} topic + * @property {OffsetCommitTopicPartition[]} partitions + * + * @typedef {Object} OffsetCommitTopicPartition + * @property {number} partition + * @property {number} offset + * @property {string} [metadata] + * + * @returns {Promise} + */ + async txnOffsetCommit({ transactionalId, groupId, producerId, producerEpoch, topics }) { + const txnOffsetCommit = this.lookupRequest(apiKeys.TxnOffsetCommit, requests.TxnOffsetCommit) + return await this[PRIVATE.SEND_REQUEST]( + txnOffsetCommit({ transactionalId, groupId, producerId, producerEpoch, topics }) + ) + } + + /** + * Send an `EndTxn` request to indicate transaction should be committed or aborted. + * + * Request should be made to the transaction coordinator. + * @public + * @param {object} request + * @param {string} request.transactionalId The transactional id corresponding to the transaction. + * @param {number} request.producerId Current producer id in use by the transactional id. + * @param {number} request.producerEpoch Current epoch associated with the producer id. + * @param {boolean} request.transactionResult The result of the transaction (false = ABORT, true = COMMIT) + * @returns {Promise} + */ + async endTxn({ transactionalId, producerId, producerEpoch, transactionResult }) { + const endTxn = this.lookupRequest(apiKeys.EndTxn, requests.EndTxn) + return await this[PRIVATE.SEND_REQUEST]( + endTxn({ transactionalId, producerId, producerEpoch, transactionResult }) + ) + } + + /** + * Send request for list of groups + * @public + * @returns {Promise} + */ + async listGroups() { + const listGroups = this.lookupRequest(apiKeys.ListGroups, requests.ListGroups) + return await this[PRIVATE.SEND_REQUEST](listGroups()) + } + + /** + * Send request to delete groups + * @param {string[]} groupIds + * @public + * @returns {Promise} + */ + async deleteGroups(groupIds) { + const deleteGroups = this.lookupRequest(apiKeys.DeleteGroups, requests.DeleteGroups) + return await this[PRIVATE.SEND_REQUEST](deleteGroups(groupIds)) + } + + /** + * Send request to delete records + * @public + * @param {object} request + * @param {TopicPartitionRecords[]} request.topics + * [ + * { + * topic: 'my-topic-name', + * partitions: [ + * { partition: 0, offset 2 }, + * { partition: 1, offset 4 }, + * ], + * } + * ] + * @returns {Promise} example: + * { + * throttleTime: 0 + * [ + * { + * topic: 'my-topic-name', + * partitions: [ + * { partition: 0, lowWatermark: '2n', errorCode: 0 }, + * { partition: 1, lowWatermark: '4n', errorCode: 0 }, + * ], + * }, + * ] + * } + * + * @typedef {object} TopicPartitionRecords + * @property {string} topic + * @property {PartitionRecord[]} partitions + * + * @typedef {object} PartitionRecord + * @property {number} partition + * @property {number} offset + */ + async deleteRecords({ topics }) { + const deleteRecords = this.lookupRequest(apiKeys.DeleteRecords, requests.DeleteRecords) + return await this[PRIVATE.SEND_REQUEST](deleteRecords({ topics })) + } + + /** + * @public + * @param {object} request + * @param {import("../../types").AclEntry[]} request.acl e.g: + * [ + * { + * resourceType: AclResourceTypes.TOPIC, + * resourceName: 'topic-name', + * resourcePatternType: ResourcePatternTypes.LITERAL, + * principal: 'User:bob', + * host: '*', + * operation: AclOperationTypes.ALL, + * permissionType: AclPermissionTypes.DENY, + * } + * ] + * @returns {Promise} + */ + async createAcls({ acl }) { + const createAcls = this.lookupRequest(apiKeys.CreateAcls, requests.CreateAcls) + return await this[PRIVATE.SEND_REQUEST](createAcls({ creations: acl })) + } + + /** + * @public + * @param {import("../../types").AclEntry} aclEntry + * @returns {Promise} + */ + async describeAcls({ + resourceType, + resourceName, + resourcePatternType, + principal, + host, + operation, + permissionType, + }) { + const describeAcls = this.lookupRequest(apiKeys.DescribeAcls, requests.DescribeAcls) + return await this[PRIVATE.SEND_REQUEST]( + describeAcls({ + resourceType, + resourceName, + resourcePatternType, + principal, + host, + operation, + permissionType, + }) + ) + } + + /** + * @public + * @param {Object} request + * @param {import("../../types").AclEntry[]} request.filters + * @returns {Promise} + */ + async deleteAcls({ filters }) { + const deleteAcls = this.lookupRequest(apiKeys.DeleteAcls, requests.DeleteAcls) + return await this[PRIVATE.SEND_REQUEST](deleteAcls({ filters })) + } + + /** + * @public + * @param {Object} request + * @param {import("../../types").PartitionReassignment[]} request.topics + * @param {number} [request.timeout] + * @returns {Promise} + */ + async alterPartitionReassignments({ topics, timeout }) { + const alterPartitionReassignments = this.lookupRequest( + apiKeys.AlterPartitionReassignments, + requests.AlterPartitionReassignments + ) + return await this[PRIVATE.SEND_REQUEST](alterPartitionReassignments({ topics, timeout })) + } + + /** + * @public + * @param {Object} request + * @param {import("../../types").TopicPartitions[]} request.topics can be null + * @param {number} [request.timeout] + * @returns {Promise} + */ + async listPartitionReassignments({ topics = null, timeout }) { + const listPartitionReassignments = this.lookupRequest( + apiKeys.ListPartitionReassignments, + requests.ListPartitionReassignments + ) + return await this[PRIVATE.SEND_REQUEST](listPartitionReassignments({ topics, timeout })) + } + + /** + * @private + */ + async [PRIVATE.SEND_REQUEST](protocolRequest) { + try { + return await this.connectionPool.send(protocolRequest) + } catch (e) { + if (e.name === 'KafkaJSConnectionClosedError') { + await this.disconnect() + } + + throw e + } + } +} diff --git a/node_modules/kafkajs/src/broker/saslAuthenticator/awsIam.js b/node_modules/kafkajs/src/broker/saslAuthenticator/awsIam.js new file mode 100644 index 0000000..45e166f --- /dev/null +++ b/node_modules/kafkajs/src/broker/saslAuthenticator/awsIam.js @@ -0,0 +1,37 @@ +const { request, response } = require('../../protocol/sasl/awsIam') +const { KafkaJSSASLAuthenticationError } = require('../../errors') + +const awsIAMAuthenticatorProvider = sasl => ({ host, port, logger, saslAuthenticate }) => { + return { + authenticate: async () => { + if (!sasl.authorizationIdentity) { + throw new KafkaJSSASLAuthenticationError('SASL AWS-IAM: Missing authorizationIdentity') + } + if (!sasl.accessKeyId) { + throw new KafkaJSSASLAuthenticationError('SASL AWS-IAM: Missing accessKeyId') + } + if (!sasl.secretAccessKey) { + throw new KafkaJSSASLAuthenticationError('SASL AWS-IAM: Missing secretAccessKey') + } + if (!sasl.sessionToken) { + sasl.sessionToken = '' + } + + const broker = `${host}:${port}` + + try { + logger.debug('Authenticate with SASL AWS-IAM', { broker }) + await saslAuthenticate({ request: request(sasl), response }) + logger.debug('SASL AWS-IAM authentication successful', { broker }) + } catch (e) { + const error = new KafkaJSSASLAuthenticationError( + `SASL AWS-IAM authentication failed: ${e.message}` + ) + logger.error(error.message, { broker }) + throw error + } + }, + } +} + +module.exports = awsIAMAuthenticatorProvider diff --git a/node_modules/kafkajs/src/broker/saslAuthenticator/index.js b/node_modules/kafkajs/src/broker/saslAuthenticator/index.js new file mode 100644 index 0000000..ddd1f90 --- /dev/null +++ b/node_modules/kafkajs/src/broker/saslAuthenticator/index.js @@ -0,0 +1,82 @@ +const { requests, lookup } = require('../../protocol/requests') +const apiKeys = require('../../protocol/requests/apiKeys') +const plainAuthenticatorProvider = require('./plain') +const scram256AuthenticatorProvider = require('./scram256') +const scram512AuthenticatorProvider = require('./scram512') +const awsIAMAuthenticatorProvider = require('./awsIam') +const oauthBearerAuthenticatorProvider = require('./oauthBearer') +const { KafkaJSSASLAuthenticationError } = require('../../errors') + +const BUILT_IN_AUTHENTICATION_PROVIDERS = { + AWS: awsIAMAuthenticatorProvider, + PLAIN: plainAuthenticatorProvider, + OAUTHBEARER: oauthBearerAuthenticatorProvider, + 'SCRAM-SHA-256': scram256AuthenticatorProvider, + 'SCRAM-SHA-512': scram512AuthenticatorProvider, +} + +const UNLIMITED_SESSION_LIFETIME = '0' + +module.exports = class SASLAuthenticator { + constructor(connection, logger, versions, supportAuthenticationProtocol) { + this.connection = connection + this.logger = logger + this.sessionLifetime = UNLIMITED_SESSION_LIFETIME + + const lookupRequest = lookup(versions) + this.saslHandshake = lookupRequest(apiKeys.SaslHandshake, requests.SaslHandshake) + this.protocolAuthentication = supportAuthenticationProtocol + ? lookupRequest(apiKeys.SaslAuthenticate, requests.SaslAuthenticate) + : null + } + + async authenticate() { + const mechanism = this.connection.sasl.mechanism.toUpperCase() + const handshake = await this.connection.send(this.saslHandshake({ mechanism })) + if (!handshake.enabledMechanisms.includes(mechanism)) { + throw new KafkaJSSASLAuthenticationError( + `SASL ${mechanism} mechanism is not supported by the server` + ) + } + + const saslAuthenticate = async ({ request, response }) => { + if (this.protocolAuthentication) { + const requestAuthBytes = await request.encode() + const authResponse = await this.connection.send( + this.protocolAuthentication({ authBytes: requestAuthBytes }) + ) + + // `0` is a string because `sessionLifetimeMs` is an int64 encoded as string. + // This is not present in SaslAuthenticateV0, so we default to `"0"` + this.sessionLifetime = authResponse.sessionLifetimeMs || UNLIMITED_SESSION_LIFETIME + + if (!response) { + return + } + + const { authBytes: responseAuthBytes } = authResponse + const payloadDecoded = await response.decode(responseAuthBytes) + return response.parse(payloadDecoded) + } + + return this.connection.sendAuthRequest({ request, response }) + } + + if ( + !this.connection.sasl.authenticationProvider && + Object.keys(BUILT_IN_AUTHENTICATION_PROVIDERS).includes(mechanism) + ) { + this.connection.sasl.authenticationProvider = BUILT_IN_AUTHENTICATION_PROVIDERS[mechanism]( + this.connection.sasl + ) + } + await this.connection.sasl + .authenticationProvider({ + host: this.connection.host, + port: this.connection.port, + logger: this.logger.namespace(`SaslAuthenticator-${mechanism}`), + saslAuthenticate, + }) + .authenticate() + } +} diff --git a/node_modules/kafkajs/src/broker/saslAuthenticator/oauthBearer.js b/node_modules/kafkajs/src/broker/saslAuthenticator/oauthBearer.js new file mode 100644 index 0000000..086a17a --- /dev/null +++ b/node_modules/kafkajs/src/broker/saslAuthenticator/oauthBearer.js @@ -0,0 +1,50 @@ +/** + * The sasl object must include a property named oauthBearerProvider, an + * async function that is used to return the OAuth bearer token. + * + * The OAuth bearer token must be an object with properties value and + * (optionally) extensions, that will be sent during the SASL/OAUTHBEARER + * request. + * + * The implementation of the oauthBearerProvider must take care that tokens are + * reused and refreshed when appropriate. + */ + +const { request } = require('../../protocol/sasl/oauthBearer') +const { KafkaJSSASLAuthenticationError } = require('../../errors') + +const oauthBearerAuthenticatorProvider = sasl => ({ host, port, logger, saslAuthenticate }) => { + return { + authenticate: async () => { + const { oauthBearerProvider } = sasl + + if (oauthBearerProvider == null) { + throw new KafkaJSSASLAuthenticationError( + 'SASL OAUTHBEARER: Missing OAuth bearer token provider' + ) + } + + const oauthBearerToken = await oauthBearerProvider() + + if (oauthBearerToken.value == null) { + throw new KafkaJSSASLAuthenticationError('SASL OAUTHBEARER: Invalid OAuth bearer token') + } + + const broker = `${host}:${port}` + + try { + logger.debug('Authenticate with SASL OAUTHBEARER', { broker }) + await saslAuthenticate({ request: await request(sasl, oauthBearerToken) }) + logger.debug('SASL OAUTHBEARER authentication successful', { broker }) + } catch (e) { + const error = new KafkaJSSASLAuthenticationError( + `SASL OAUTHBEARER authentication failed: ${e.message}` + ) + logger.error(error.message, { broker }) + throw error + } + }, + } +} + +module.exports = oauthBearerAuthenticatorProvider diff --git a/node_modules/kafkajs/src/broker/saslAuthenticator/plain.js b/node_modules/kafkajs/src/broker/saslAuthenticator/plain.js new file mode 100644 index 0000000..fb63463 --- /dev/null +++ b/node_modules/kafkajs/src/broker/saslAuthenticator/plain.js @@ -0,0 +1,28 @@ +const { request, response } = require('../../protocol/sasl/plain') +const { KafkaJSSASLAuthenticationError } = require('../../errors') + +const plainAuthenticatorProvider = sasl => ({ host, port, logger, saslAuthenticate }) => { + return { + authenticate: async () => { + if (sasl.username == null || sasl.password == null) { + throw new KafkaJSSASLAuthenticationError('SASL Plain: Invalid username or password') + } + + const broker = `${host}:${port}` + + try { + logger.debug('Authenticate with SASL PLAIN', { broker }) + await saslAuthenticate({ request: request(sasl), response }) + logger.debug('SASL PLAIN authentication successful', { broker }) + } catch (e) { + const error = new KafkaJSSASLAuthenticationError( + `SASL PLAIN authentication failed: ${e.message}` + ) + logger.error(error.message, { broker }) + throw error + } + }, + } +} + +module.exports = plainAuthenticatorProvider diff --git a/node_modules/kafkajs/src/broker/saslAuthenticator/scram.js b/node_modules/kafkajs/src/broker/saslAuthenticator/scram.js new file mode 100644 index 0000000..355fdbc --- /dev/null +++ b/node_modules/kafkajs/src/broker/saslAuthenticator/scram.js @@ -0,0 +1,325 @@ +const crypto = require('crypto') +const scram = require('../../protocol/sasl/scram') +const { KafkaJSSASLAuthenticationError, KafkaJSNonRetriableError } = require('../../errors') + +const GS2_HEADER = 'n,,' + +const EQUAL_SIGN_REGEX = /=/g +const COMMA_SIGN_REGEX = /,/g + +const URLSAFE_BASE64_PLUS_REGEX = /\+/g +const URLSAFE_BASE64_SLASH_REGEX = /\//g +const URLSAFE_BASE64_TRAILING_EQUAL_REGEX = /=+$/ + +const HMAC_CLIENT_KEY = 'Client Key' +const HMAC_SERVER_KEY = 'Server Key' + +const DIGESTS = { + SHA256: { + length: 32, + type: 'sha256', + minIterations: 4096, + }, + SHA512: { + length: 64, + type: 'sha512', + minIterations: 4096, + }, +} + +const encode64 = str => Buffer.from(str).toString('base64') + +class SCRAM { + /** + * From https://tools.ietf.org/html/rfc5802#section-5.1 + * + * The characters ',' or '=' in usernames are sent as '=2C' and + * '=3D' respectively. If the server receives a username that + * contains '=' not followed by either '2C' or '3D', then the + * server MUST fail the authentication. + * + * @returns {String} + */ + static sanitizeString(str) { + return str.replace(EQUAL_SIGN_REGEX, '=3D').replace(COMMA_SIGN_REGEX, '=2C') + } + + /** + * In cryptography, a nonce is an arbitrary number that can be used just once. + * It is similar in spirit to a nonce * word, hence the name. It is often a random or pseudo-random + * number issued in an authentication protocol to * ensure that old communications cannot be reused + * in replay attacks. + * + * @returns {String} + */ + static nonce() { + return crypto + .randomBytes(16) + .toString('base64') + .replace(URLSAFE_BASE64_PLUS_REGEX, '-') // make it url safe + .replace(URLSAFE_BASE64_SLASH_REGEX, '_') + .replace(URLSAFE_BASE64_TRAILING_EQUAL_REGEX, '') + .toString('ascii') + } + + /** + * Hi() is, essentially, PBKDF2 [RFC2898] with HMAC() as the + * pseudorandom function (PRF) and with dkLen == output length of + * HMAC() == output length of H() + * + * @returns {Promise} + */ + static hi(password, salt, iterations, digestDefinition) { + return new Promise((resolve, reject) => { + crypto.pbkdf2( + password, + salt, + iterations, + digestDefinition.length, + digestDefinition.type, + (err, derivedKey) => (err ? reject(err) : resolve(derivedKey)) + ) + }) + } + + /** + * Apply the exclusive-or operation to combine the octet string + * on the left of this operator with the octet string on the right of + * this operator. The length of the output and each of the two + * inputs will be the same for this use + * + * @returns {Buffer} + */ + static xor(left, right) { + const bufferA = Buffer.from(left) + const bufferB = Buffer.from(right) + const length = Buffer.byteLength(bufferA) + + if (length !== Buffer.byteLength(bufferB)) { + throw new KafkaJSNonRetriableError('Buffers must be of the same length') + } + + const result = [] + for (let i = 0; i < length; i++) { + result.push(bufferA[i] ^ bufferB[i]) + } + + return Buffer.from(result) + } + + /** + * @param {SASLOptions} sasl + * @param {Logger} logger + * @param {Function} saslAuthenticate + * @param {DigestDefinition} digestDefinition + */ + constructor(sasl, host, port, logger, saslAuthenticate, digestDefinition) { + this.sasl = sasl + this.host = host + this.port = port + this.logger = logger + this.saslAuthenticate = saslAuthenticate + this.digestDefinition = digestDefinition + + const digestType = digestDefinition.type.toUpperCase() + this.PREFIX = `SASL SCRAM ${digestType} authentication` + + this.currentNonce = SCRAM.nonce() + } + + async authenticate() { + const { PREFIX } = this + const broker = `${this.host}:${this.port}` + + if (this.sasl.username == null || this.sasl.password == null) { + throw new KafkaJSSASLAuthenticationError(`${this.PREFIX}: Invalid username or password`) + } + + try { + this.logger.debug('Exchanging first client message', { broker }) + const clientMessageResponse = await this.sendClientFirstMessage() + + this.logger.debug('Sending final message', { broker }) + const finalResponse = await this.sendClientFinalMessage(clientMessageResponse) + + if (finalResponse.e) { + throw new Error(finalResponse.e) + } + + const serverKey = await this.serverKey(clientMessageResponse) + const serverSignature = this.serverSignature(serverKey, clientMessageResponse) + + if (finalResponse.v !== serverSignature) { + throw new Error('Invalid server signature in server final message') + } + + this.logger.debug(`${PREFIX} successful`, { broker }) + } catch (e) { + const error = new KafkaJSSASLAuthenticationError(`${PREFIX} failed: ${e.message}`) + this.logger.error(error.message, { broker }) + throw error + } + } + + /** + * @private + */ + async sendClientFirstMessage() { + const clientFirstMessage = `${GS2_HEADER}${this.firstMessageBare()}` + const request = scram.firstMessage.request({ clientFirstMessage }) + const response = scram.firstMessage.response + + return this.saslAuthenticate({ + request, + response, + }) + } + + /** + * @private + */ + async sendClientFinalMessage(clientMessageResponse) { + const { PREFIX } = this + const iterations = parseInt(clientMessageResponse.i, 10) + const { minIterations } = this.digestDefinition + + if (!clientMessageResponse.r.startsWith(this.currentNonce)) { + throw new KafkaJSSASLAuthenticationError( + `${PREFIX} failed: Invalid server nonce, it does not start with the client nonce` + ) + } + + if (iterations < minIterations) { + throw new KafkaJSSASLAuthenticationError( + `${PREFIX} failed: Requested iterations ${iterations} is less than the minimum ${minIterations}` + ) + } + + const finalMessageWithoutProof = this.finalMessageWithoutProof(clientMessageResponse) + const clientProof = await this.clientProof(clientMessageResponse) + const finalMessage = `${finalMessageWithoutProof},p=${clientProof}` + const request = scram.finalMessage.request({ finalMessage }) + const response = scram.finalMessage.response + + return this.saslAuthenticate({ + request, + response, + }) + } + + /** + * @private + */ + async clientProof(clientMessageResponse) { + const clientKey = await this.clientKey(clientMessageResponse) + const storedKey = this.H(clientKey) + const clientSignature = this.clientSignature(storedKey, clientMessageResponse) + return encode64(SCRAM.xor(clientKey, clientSignature)) + } + + /** + * @private + */ + async clientKey(clientMessageResponse) { + const saltedPassword = await this.saltPassword(clientMessageResponse) + return this.HMAC(saltedPassword, HMAC_CLIENT_KEY) + } + + /** + * @private + */ + async serverKey(clientMessageResponse) { + const saltedPassword = await this.saltPassword(clientMessageResponse) + return this.HMAC(saltedPassword, HMAC_SERVER_KEY) + } + + /** + * @private + */ + clientSignature(storedKey, clientMessageResponse) { + return this.HMAC(storedKey, this.authMessage(clientMessageResponse)) + } + + /** + * @private + */ + serverSignature(serverKey, clientMessageResponse) { + return encode64(this.HMAC(serverKey, this.authMessage(clientMessageResponse))) + } + + /** + * @private + */ + authMessage(clientMessageResponse) { + return [ + this.firstMessageBare(), + clientMessageResponse.original, + this.finalMessageWithoutProof(clientMessageResponse), + ].join(',') + } + + /** + * @private + */ + async saltPassword(clientMessageResponse) { + const salt = Buffer.from(clientMessageResponse.s, 'base64') + const iterations = parseInt(clientMessageResponse.i, 10) + return SCRAM.hi(this.encodedPassword(), salt, iterations, this.digestDefinition) + } + + /** + * @private + */ + firstMessageBare() { + return `n=${this.encodedUsername()},r=${this.currentNonce}` + } + + /** + * @private + */ + finalMessageWithoutProof(clientMessageResponse) { + const rnonce = clientMessageResponse.r + return `c=${encode64(GS2_HEADER)},r=${rnonce}` + } + + /** + * @private + */ + encodedUsername() { + const { username } = this.sasl + return SCRAM.sanitizeString(username).toString('utf-8') + } + + /** + * @private + */ + encodedPassword() { + const { password } = this.sasl + return password.toString('utf-8') + } + + /** + * @private + */ + H(data) { + return crypto + .createHash(this.digestDefinition.type) + .update(data) + .digest() + } + + /** + * @private + */ + HMAC(key, data) { + return crypto + .createHmac(this.digestDefinition.type, key) + .update(data) + .digest() + } +} + +module.exports = { + DIGESTS, + SCRAM, +} diff --git a/node_modules/kafkajs/src/broker/saslAuthenticator/scram256.js b/node_modules/kafkajs/src/broker/saslAuthenticator/scram256.js new file mode 100644 index 0000000..34030f4 --- /dev/null +++ b/node_modules/kafkajs/src/broker/saslAuthenticator/scram256.js @@ -0,0 +1,10 @@ +const { SCRAM, DIGESTS } = require('./scram') + +const scram256AuthenticatorProvider = sasl => ({ host, port, logger, saslAuthenticate }) => { + const scram = new SCRAM(sasl, host, port, logger, saslAuthenticate, DIGESTS.SHA256) + return { + authenticate: async () => await scram.authenticate(), + } +} + +module.exports = scram256AuthenticatorProvider diff --git a/node_modules/kafkajs/src/broker/saslAuthenticator/scram512.js b/node_modules/kafkajs/src/broker/saslAuthenticator/scram512.js new file mode 100644 index 0000000..961ec0d --- /dev/null +++ b/node_modules/kafkajs/src/broker/saslAuthenticator/scram512.js @@ -0,0 +1,10 @@ +const { SCRAM, DIGESTS } = require('./scram') + +const scram512AuthenticatorProvider = sasl => ({ host, port, logger, saslAuthenticate }) => { + const scram = new SCRAM(sasl, host, port, logger, saslAuthenticate, DIGESTS.SHA512) + return { + authenticate: async () => await scram.authenticate(), + } +} + +module.exports = scram512AuthenticatorProvider diff --git a/node_modules/kafkajs/src/cluster/brokerPool.js b/node_modules/kafkajs/src/cluster/brokerPool.js new file mode 100644 index 0000000..a35d2dc --- /dev/null +++ b/node_modules/kafkajs/src/cluster/brokerPool.js @@ -0,0 +1,349 @@ +const Broker = require('../broker') +const createRetry = require('../retry') +const shuffle = require('../utils/shuffle') +const arrayDiff = require('../utils/arrayDiff') +const { KafkaJSBrokerNotFound, KafkaJSProtocolError } = require('../errors') + +const { keys, assign, values } = Object +const hasBrokerBeenReplaced = (broker, { host, port, rack }) => + broker.connectionPool.host !== host || + broker.connectionPool.port !== port || + broker.connectionPool.rack !== rack + +module.exports = class BrokerPool { + /** + * @param {object} options + * @param {import("./connectionPoolBuilder").ConnectionPoolBuilder} options.connectionPoolBuilder + * @param {import("../../types").Logger} options.logger + * @param {import("../../types").RetryOptions} [options.retry] + * @param {boolean} [options.allowAutoTopicCreation] + * @param {number} [options.authenticationTimeout] + * @param {number} [options.metadataMaxAge] + */ + constructor({ + connectionPoolBuilder, + logger, + retry, + allowAutoTopicCreation, + authenticationTimeout, + metadataMaxAge, + }) { + this.rootLogger = logger + this.connectionPoolBuilder = connectionPoolBuilder + this.metadataMaxAge = metadataMaxAge || 0 + this.logger = logger.namespace('BrokerPool') + this.retrier = createRetry(assign({}, retry)) + + this.createBroker = options => + new Broker({ + allowAutoTopicCreation, + authenticationTimeout, + ...options, + }) + + this.brokers = {} + /** @type {Broker | undefined} */ + this.seedBroker = undefined + /** @type {import("../../types").BrokerMetadata | null} */ + this.metadata = null + this.metadataExpireAt = null + this.versions = null + } + + /** + * @public + * @returns {Boolean} + */ + hasConnectedBrokers() { + const brokers = values(this.brokers) + return ( + !!brokers.find(broker => broker.isConnected()) || + (this.seedBroker ? this.seedBroker.isConnected() : false) + ) + } + + async createSeedBroker() { + if (this.seedBroker) { + await this.seedBroker.disconnect() + } + + const connectionPool = await this.connectionPoolBuilder.build() + + this.seedBroker = this.createBroker({ + connectionPool, + logger: this.rootLogger, + }) + } + + /** + * @public + * @returns {Promise} + */ + async connect() { + if (this.hasConnectedBrokers()) { + return + } + + if (!this.seedBroker) { + await this.createSeedBroker() + } + + return this.retrier(async (bail, retryCount, retryTime) => { + try { + await this.seedBroker.connect() + this.versions = this.seedBroker.versions + } catch (e) { + if (e.name === 'KafkaJSConnectionError' || e.type === 'ILLEGAL_SASL_STATE') { + // Connection builder will always rotate the seed broker + await this.createSeedBroker() + this.logger.error( + `Failed to connect to seed broker, trying another broker from the list: ${e.message}`, + { retryCount, retryTime } + ) + } else { + this.logger.error(e.message, { retryCount, retryTime }) + } + + if (e.retriable) throw e + bail(e) + } + }) + } + + /** + * @public + * @returns {Promise} + */ + async disconnect() { + this.seedBroker && (await this.seedBroker.disconnect()) + await Promise.all(values(this.brokers).map(broker => broker.disconnect())) + + this.brokers = {} + this.metadata = null + this.versions = null + } + + /** + * @public + * @param {Object} destination + * @param {string} destination.host + * @param {number} destination.port + */ + removeBroker({ host, port }) { + const removedBroker = values(this.brokers).find( + broker => broker.connectionPool.host === host && broker.connectionPool.port === port + ) + + if (removedBroker) { + delete this.brokers[removedBroker.nodeId] + this.metadataExpireAt = null + + if (this.seedBroker.nodeId === removedBroker.nodeId) { + this.seedBroker = shuffle(values(this.brokers))[0] + } + } + } + + /** + * @public + * @param {Array} topics + * @returns {Promise} + */ + async refreshMetadata(topics) { + const broker = await this.findConnectedBroker() + const { host: seedHost, port: seedPort } = this.seedBroker.connectionPool + + return this.retrier(async (bail, retryCount, retryTime) => { + try { + this.metadata = await broker.metadata(topics) + this.metadataExpireAt = Date.now() + this.metadataMaxAge + + const replacedBrokers = [] + + this.brokers = await this.metadata.brokers.reduce( + async (resultPromise, { nodeId, host, port, rack }) => { + const result = await resultPromise + + if (result[nodeId]) { + if (!hasBrokerBeenReplaced(result[nodeId], { host, port, rack })) { + return result + } + + replacedBrokers.push(result[nodeId]) + } + + if (host === seedHost && port === seedPort) { + this.seedBroker.nodeId = nodeId + this.seedBroker.connectionPool.rack = rack + return assign(result, { + [nodeId]: this.seedBroker, + }) + } + + return assign(result, { + [nodeId]: this.createBroker({ + logger: this.rootLogger, + versions: this.versions, + connectionPool: await this.connectionPoolBuilder.build({ host, port, rack }), + nodeId, + }), + }) + }, + this.brokers + ) + + const freshBrokerIds = this.metadata.brokers.map(({ nodeId }) => `${nodeId}`).sort() + const currentBrokerIds = keys(this.brokers).sort() + const unusedBrokerIds = arrayDiff(currentBrokerIds, freshBrokerIds) + + const brokerDisconnects = unusedBrokerIds.map(nodeId => { + const broker = this.brokers[nodeId] + return broker.disconnect().then(() => { + delete this.brokers[nodeId] + }) + }) + + const replacedBrokersDisconnects = replacedBrokers.map(broker => broker.disconnect()) + await Promise.all([...brokerDisconnects, ...replacedBrokersDisconnects]) + } catch (e) { + if (e.type === 'LEADER_NOT_AVAILABLE') { + throw e + } + + bail(e) + } + }) + } + + /** + * Only refreshes metadata if the data is stale according to the `metadataMaxAge` param or does not contain information about the provided topics + * + * @public + * @param {Array} topics + * @returns {Promise} + */ + async refreshMetadataIfNecessary(topics) { + const shouldRefresh = + this.metadata == null || + this.metadataExpireAt == null || + Date.now() > this.metadataExpireAt || + !topics.every(topic => + this.metadata.topicMetadata.some(topicMetadata => topicMetadata.topic === topic) + ) + + if (shouldRefresh) { + return this.refreshMetadata(topics) + } + } + + /** @type {() => string[]} */ + getNodeIds() { + return keys(this.brokers) + } + + /** + * @public + * @param {object} options + * @param {string} options.nodeId + * @returns {Promise} + */ + async findBroker({ nodeId }) { + const broker = this.brokers[nodeId] + + if (!broker) { + throw new KafkaJSBrokerNotFound(`Broker ${nodeId} not found in the cached metadata`) + } + + await this.connectBroker(broker) + return broker + } + + /** + * @public + * @param {(params: { nodeId: string, broker: Broker }) => Promise} callback + * @returns {Promise} + * @template T + */ + async withBroker(callback) { + const brokers = shuffle(keys(this.brokers)) + if (brokers.length === 0) { + throw new KafkaJSBrokerNotFound('No brokers in the broker pool') + } + + for (const nodeId of brokers) { + const broker = await this.findBroker({ nodeId }) + try { + return await callback({ nodeId, broker }) + } catch (e) {} + } + + return null + } + + /** + * @public + * @returns {Promise} + */ + async findConnectedBroker() { + const nodeIds = shuffle(keys(this.brokers)) + const connectedBrokerId = nodeIds.find(nodeId => this.brokers[nodeId].isConnected()) + + if (connectedBrokerId) { + return await this.findBroker({ nodeId: connectedBrokerId }) + } + + // Cycle through the nodes until one connects + for (const nodeId of nodeIds) { + try { + return await this.findBroker({ nodeId }) + } catch (e) {} + } + + // Failed to connect to all known brokers, metadata might be old + await this.connect() + return this.seedBroker + } + + /** + * @private + * @param {Broker} broker + * @returns {Promise} + */ + async connectBroker(broker) { + if (broker.isConnected()) { + return + } + + return this.retrier(async (bail, retryCount, retryTime) => { + try { + await broker.connect() + } catch (e) { + if (e.name === 'KafkaJSConnectionError' || e.type === 'ILLEGAL_SASL_STATE') { + await broker.disconnect() + } + + // To avoid reconnecting to an unavailable host, we bail on connection errors + // and refresh metadata on a higher level before reconnecting + if (e.name === 'KafkaJSConnectionError') { + return bail(e) + } + + if (e.type === 'ILLEGAL_SASL_STATE') { + // Rebuild the connection pool since it can't recover from illegal SASL state + broker.connectionPool = await this.connectionPoolBuilder.build({ + host: broker.connectionPool.host, + port: broker.connectionPool.port, + rack: broker.connectionPool.rack, + }) + + this.logger.error(`Failed to connect to broker, reconnecting`, { retryCount, retryTime }) + throw new KafkaJSProtocolError(e, { retriable: true }) + } + + if (e.retriable) throw e + this.logger.error(e, { retryCount, retryTime, stack: e.stack }) + bail(e) + } + }) + } +} diff --git a/node_modules/kafkajs/src/cluster/connectionPoolBuilder.js b/node_modules/kafkajs/src/cluster/connectionPoolBuilder.js new file mode 100644 index 0000000..b29f957 --- /dev/null +++ b/node_modules/kafkajs/src/cluster/connectionPoolBuilder.js @@ -0,0 +1,117 @@ +const { KafkaJSConnectionError, KafkaJSNonRetriableError } = require('../errors') +const ConnectionPool = require('../network/connectionPool') + +/** + * @typedef {Object} ConnectionPoolBuilder + * @property {(destination?: { host?: string, port?: number, rack?: string }) => Promise} build + */ + +/** + * @param {Object} options + * @param {import("../../types").ISocketFactory} [options.socketFactory] + * @param {string[]|(() => string[])} options.brokers + * @param {Object} [options.ssl] + * @param {Object} [options.sasl] + * @param {string} options.clientId + * @param {number} options.requestTimeout + * @param {boolean} [options.enforceRequestTimeout] + * @param {number} [options.connectionTimeout] + * @param {number} [options.maxInFlightRequests] + * @param {import("../../types").RetryOptions} [options.retry] + * @param {import("../../types").Logger} options.logger + * @param {import("../instrumentation/emitter")} [options.instrumentationEmitter] + * @param {number} [options.reauthenticationThreshold] + * @returns {ConnectionPoolBuilder} + */ +module.exports = ({ + socketFactory, + brokers, + ssl, + sasl, + clientId, + requestTimeout, + enforceRequestTimeout, + connectionTimeout, + maxInFlightRequests, + logger, + instrumentationEmitter = null, + reauthenticationThreshold, +}) => { + let index = 0 + + const isValidBroker = broker => { + return broker && typeof broker === 'string' && broker.length > 0 + } + + const validateBrokers = brokers => { + if (!brokers) { + throw new KafkaJSNonRetriableError(`Failed to connect: brokers should not be null`) + } + + if (Array.isArray(brokers)) { + if (!brokers.length) { + throw new KafkaJSNonRetriableError(`Failed to connect: brokers array is empty`) + } + + brokers.forEach((broker, index) => { + if (!isValidBroker(broker)) { + throw new KafkaJSNonRetriableError( + `Failed to connect: broker at index ${index} is invalid "${typeof broker}"` + ) + } + }) + } + } + + const getBrokers = async () => { + let list + + if (typeof brokers === 'function') { + try { + list = await brokers() + } catch (e) { + const wrappedError = new KafkaJSConnectionError( + `Failed to connect: "config.brokers" threw: ${e.message}` + ) + wrappedError.stack = `${wrappedError.name}\n Caused by: ${e.stack}` + throw wrappedError + } + } else { + list = brokers + } + + validateBrokers(list) + + return list + } + + return { + build: async ({ host, port, rack } = {}) => { + if (!host) { + const list = await getBrokers() + + const randomBroker = list[index++ % list.length] + + host = randomBroker.split(':')[0] + port = Number(randomBroker.split(':')[1]) + } + + return new ConnectionPool({ + host, + port, + rack, + sasl, + ssl, + clientId, + socketFactory, + connectionTimeout, + requestTimeout, + enforceRequestTimeout, + maxInFlightRequests, + instrumentationEmitter, + logger, + reauthenticationThreshold, + }) + }, + } +} diff --git a/node_modules/kafkajs/src/cluster/index.js b/node_modules/kafkajs/src/cluster/index.js new file mode 100644 index 0000000..af0a919 --- /dev/null +++ b/node_modules/kafkajs/src/cluster/index.js @@ -0,0 +1,539 @@ +const BrokerPool = require('./brokerPool') +const Lock = require('../utils/lock') +const sharedPromiseTo = require('../utils/sharedPromiseTo') +const createRetry = require('../retry') +const connectionPoolBuilder = require('./connectionPoolBuilder') +const { EARLIEST_OFFSET, LATEST_OFFSET } = require('../constants') +const { + KafkaJSError, + KafkaJSBrokerNotFound, + KafkaJSMetadataNotLoaded, + KafkaJSTopicMetadataNotLoaded, + KafkaJSGroupCoordinatorNotFound, +} = require('../errors') +const COORDINATOR_TYPES = require('../protocol/coordinatorTypes') + +const { keys } = Object + +const mergeTopics = (obj, { topic, partitions }) => ({ + ...obj, + [topic]: [...(obj[topic] || []), ...partitions], +}) + +const PRIVATE = { + CONNECT: Symbol('private:Cluster:connect'), + REFRESH_METADATA: Symbol('private:Cluster:refreshMetadata'), + REFRESH_METADATA_IF_NECESSARY: Symbol('private:Cluster:refreshMetadataIfNecessary'), + FIND_CONTROLLER_BROKER: Symbol('private:Cluster:findControllerBroker'), +} + +module.exports = class Cluster { + /** + * @param {Object} options + * @param {Array} options.brokers example: ['127.0.0.1:9092', '127.0.0.1:9094'] + * @param {Object} options.ssl + * @param {Object} options.sasl + * @param {string} options.clientId + * @param {number} options.connectionTimeout - in milliseconds + * @param {number} options.authenticationTimeout - in milliseconds + * @param {number} options.reauthenticationThreshold - in milliseconds + * @param {number} [options.requestTimeout=30000] - in milliseconds + * @param {boolean} [options.enforceRequestTimeout] + * @param {number} options.metadataMaxAge - in milliseconds + * @param {boolean} options.allowAutoTopicCreation + * @param {number} options.maxInFlightRequests + * @param {number} options.isolationLevel + * @param {import("../../types").RetryOptions} options.retry + * @param {import("../../types").Logger} options.logger + * @param {import("../../types").ISocketFactory} options.socketFactory + * @param {Map} [options.offsets] + * @param {import("../instrumentation/emitter")} [options.instrumentationEmitter=null] + */ + constructor({ + logger: rootLogger, + socketFactory, + brokers, + ssl, + sasl, + clientId, + connectionTimeout, + authenticationTimeout, + reauthenticationThreshold, + requestTimeout = 30000, + enforceRequestTimeout, + metadataMaxAge, + retry, + allowAutoTopicCreation, + maxInFlightRequests, + isolationLevel, + instrumentationEmitter = null, + offsets = new Map(), + }) { + this.rootLogger = rootLogger + this.logger = rootLogger.namespace('Cluster') + this.retrier = createRetry(retry) + this.connectionPoolBuilder = connectionPoolBuilder({ + logger: rootLogger, + instrumentationEmitter, + socketFactory, + brokers, + ssl, + sasl, + clientId, + connectionTimeout, + requestTimeout, + enforceRequestTimeout, + maxInFlightRequests, + reauthenticationThreshold, + }) + + this.targetTopics = new Set() + this.mutatingTargetTopics = new Lock({ + description: `updating target topics`, + timeout: requestTimeout, + }) + this.isolationLevel = isolationLevel + this.brokerPool = new BrokerPool({ + connectionPoolBuilder: this.connectionPoolBuilder, + logger: this.rootLogger, + retry, + allowAutoTopicCreation, + authenticationTimeout, + metadataMaxAge, + }) + this.committedOffsetsByGroup = offsets + + this[PRIVATE.CONNECT] = sharedPromiseTo(async () => { + return await this.brokerPool.connect() + }) + + this[PRIVATE.REFRESH_METADATA] = sharedPromiseTo(async () => { + return await this.brokerPool.refreshMetadata(Array.from(this.targetTopics)) + }) + + this[PRIVATE.REFRESH_METADATA_IF_NECESSARY] = sharedPromiseTo(async () => { + return await this.brokerPool.refreshMetadataIfNecessary(Array.from(this.targetTopics)) + }) + + this[PRIVATE.FIND_CONTROLLER_BROKER] = sharedPromiseTo(async () => { + const { metadata } = this.brokerPool + + if (!metadata || metadata.controllerId == null) { + throw new KafkaJSMetadataNotLoaded('Topic metadata not loaded') + } + + const broker = await this.findBroker({ nodeId: metadata.controllerId }) + + if (!broker) { + throw new KafkaJSBrokerNotFound( + `Controller broker with id ${metadata.controllerId} not found in the cached metadata` + ) + } + + return broker + }) + } + + isConnected() { + return this.brokerPool.hasConnectedBrokers() + } + + /** + * @public + * @returns {Promise} + */ + async connect() { + await this[PRIVATE.CONNECT]() + } + + /** + * @public + * @returns {Promise} + */ + async disconnect() { + await this.brokerPool.disconnect() + } + + /** + * @public + * @param {object} destination + * @param {String} destination.host + * @param {Number} destination.port + */ + removeBroker({ host, port }) { + this.brokerPool.removeBroker({ host, port }) + } + + /** + * @public + * @returns {Promise} + */ + async refreshMetadata() { + await this[PRIVATE.REFRESH_METADATA]() + } + + /** + * @public + * @returns {Promise} + */ + async refreshMetadataIfNecessary() { + await this[PRIVATE.REFRESH_METADATA_IF_NECESSARY]() + } + + /** + * @public + * @returns {Promise} + */ + async metadata({ topics = [] } = {}) { + return this.retrier(async (bail, retryCount, retryTime) => { + try { + await this.brokerPool.refreshMetadataIfNecessary(topics) + return this.brokerPool.withBroker(async ({ broker }) => broker.metadata(topics)) + } catch (e) { + if (e.type === 'LEADER_NOT_AVAILABLE') { + throw e + } + + bail(e) + } + }) + } + + /** + * @public + * @param {string} topic + * @return {Promise} + */ + async addTargetTopic(topic) { + return this.addMultipleTargetTopics([topic]) + } + + /** + * @public + * @param {string[]} topics + * @return {Promise} + */ + async addMultipleTargetTopics(topics) { + await this.mutatingTargetTopics.acquire() + + try { + const previousSize = this.targetTopics.size + const previousTopics = new Set(this.targetTopics) + for (const topic of topics) { + this.targetTopics.add(topic) + } + + const hasChanged = previousSize !== this.targetTopics.size || !this.brokerPool.metadata + + if (hasChanged) { + try { + await this.refreshMetadata() + } catch (e) { + if ( + e.type === 'INVALID_TOPIC_EXCEPTION' || + e.type === 'UNKNOWN_TOPIC_OR_PARTITION' || + e.type === 'TOPIC_AUTHORIZATION_FAILED' + ) { + this.targetTopics = previousTopics + } + + throw e + } + } + } finally { + await this.mutatingTargetTopics.release() + } + } + + /** @type {() => string[]} */ + getNodeIds() { + return this.brokerPool.getNodeIds() + } + + /** + * @public + * @param {object} options + * @param {string} options.nodeId + * @returns {Promise} + */ + async findBroker({ nodeId }) { + try { + return await this.brokerPool.findBroker({ nodeId }) + } catch (e) { + // The client probably has stale metadata + if ( + e.name === 'KafkaJSBrokerNotFound' || + e.name === 'KafkaJSLockTimeout' || + e.name === 'KafkaJSConnectionError' + ) { + await this.refreshMetadata() + } + + throw e + } + } + + /** + * @public + * @returns {Promise} + */ + async findControllerBroker() { + return await this[PRIVATE.FIND_CONTROLLER_BROKER]() + } + + /** + * @public + * @param {string} topic + * @returns {import("../../types").PartitionMetadata[]} Example: + * [{ + * isr: [2], + * leader: 2, + * partitionErrorCode: 0, + * partitionId: 0, + * replicas: [2], + * }] + */ + findTopicPartitionMetadata(topic) { + const { metadata } = this.brokerPool + if (!metadata || !metadata.topicMetadata) { + throw new KafkaJSTopicMetadataNotLoaded('Topic metadata not loaded', { topic }) + } + + const topicMetadata = metadata.topicMetadata.find(t => t.topic === topic) + return topicMetadata ? topicMetadata.partitionMetadata : [] + } + + /** + * @public + * @param {string} topic + * @param {(number|string)[]} partitions + * @returns {Object} Object with leader and partitions. For partitions 0 and 5 + * the result could be: + * { '0': [0], '2': [5] } + * + * where the key is the nodeId. + */ + findLeaderForPartitions(topic, partitions) { + const partitionMetadata = this.findTopicPartitionMetadata(topic) + return partitions.reduce((result, id) => { + const partitionId = parseInt(id, 10) + const metadata = partitionMetadata.find(p => p.partitionId === partitionId) + + if (!metadata) { + return result + } + + if (metadata.leader === null || metadata.leader === undefined) { + throw new KafkaJSError('Invalid partition metadata', { topic, partitionId, metadata }) + } + + const { leader } = metadata + const current = result[leader] || [] + return { ...result, [leader]: [...current, partitionId] } + }, {}) + } + + /** + * @public + * @param {object} params + * @param {string} params.groupId + * @param {import("../protocol/coordinatorTypes").CoordinatorType} [params.coordinatorType=0] + * @returns {Promise} + */ + async findGroupCoordinator({ groupId, coordinatorType = COORDINATOR_TYPES.GROUP }) { + return this.retrier(async (bail, retryCount, retryTime) => { + try { + const { coordinator } = await this.findGroupCoordinatorMetadata({ + groupId, + coordinatorType, + }) + return await this.findBroker({ nodeId: coordinator.nodeId }) + } catch (e) { + // A new broker can join the cluster before we have the chance + // to refresh metadata + if (e.name === 'KafkaJSBrokerNotFound' || e.type === 'GROUP_COORDINATOR_NOT_AVAILABLE') { + this.logger.debug(`${e.message}, refreshing metadata and trying again...`, { + groupId, + retryCount, + retryTime, + }) + + await this.refreshMetadata() + throw e + } + + if (e.code === 'ECONNREFUSED') { + // During maintenance the current coordinator can go down; findBroker will + // refresh metadata and re-throw the error. findGroupCoordinator has to re-throw + // the error to go through the retry cycle. + throw e + } + + bail(e) + } + }) + } + + /** + * @public + * @param {object} params + * @param {string} params.groupId + * @param {import("../protocol/coordinatorTypes").CoordinatorType} [params.coordinatorType=0] + * @returns {Promise} + */ + async findGroupCoordinatorMetadata({ groupId, coordinatorType }) { + const brokerMetadata = await this.brokerPool.withBroker(async ({ nodeId, broker }) => { + return await this.retrier(async (bail, retryCount, retryTime) => { + try { + const brokerMetadata = await broker.findGroupCoordinator({ groupId, coordinatorType }) + this.logger.debug('Found group coordinator', { + broker: brokerMetadata.host, + nodeId: brokerMetadata.coordinator.nodeId, + }) + return brokerMetadata + } catch (e) { + this.logger.debug('Tried to find group coordinator', { + nodeId, + groupId, + error: e, + }) + + if (e.type === 'GROUP_COORDINATOR_NOT_AVAILABLE') { + this.logger.debug('Group coordinator not available, retrying...', { + nodeId, + retryCount, + retryTime, + }) + + throw e + } + + bail(e) + } + }) + }) + + if (brokerMetadata) { + return brokerMetadata + } + + throw new KafkaJSGroupCoordinatorNotFound('Failed to find group coordinator') + } + + /** + * @param {object} topicConfiguration + * @returns {number} + */ + defaultOffset({ fromBeginning }) { + return fromBeginning ? EARLIEST_OFFSET : LATEST_OFFSET + } + + /** + * @public + * @param {Array} topics + * [ + * { + * topic: 'my-topic-name', + * partitions: [{ partition: 0 }], + * fromBeginning: false + * } + * ] + * @returns {Promise} example: + * [ + * { + * topic: 'my-topic-name', + * partitions: [ + * { partition: 0, offset: '1' }, + * { partition: 1, offset: '2' }, + * { partition: 2, offset: '1' }, + * ], + * }, + * ] + */ + async fetchTopicsOffset(topics) { + const partitionsPerBroker = {} + const topicConfigurations = {} + + const addDefaultOffset = topic => partition => { + const { timestamp } = topicConfigurations[topic] + return { ...partition, timestamp } + } + + // Index all topics and partitions per leader (nodeId) + for (const topicData of topics) { + const { topic, partitions, fromBeginning, fromTimestamp } = topicData + const partitionsPerLeader = this.findLeaderForPartitions( + topic, + partitions.map(p => p.partition) + ) + const timestamp = + fromTimestamp != null ? fromTimestamp : this.defaultOffset({ fromBeginning }) + + topicConfigurations[topic] = { timestamp } + + keys(partitionsPerLeader).forEach(nodeId => { + partitionsPerBroker[nodeId] = partitionsPerBroker[nodeId] || {} + partitionsPerBroker[nodeId][topic] = partitions.filter(p => + partitionsPerLeader[nodeId].includes(p.partition) + ) + }) + } + + // Create a list of requests to fetch the offset of all partitions + const requests = keys(partitionsPerBroker).map(async nodeId => { + const broker = await this.findBroker({ nodeId }) + const partitions = partitionsPerBroker[nodeId] + + const { responses: topicOffsets } = await broker.listOffsets({ + isolationLevel: this.isolationLevel, + topics: keys(partitions).map(topic => ({ + topic, + partitions: partitions[topic].map(addDefaultOffset(topic)), + })), + }) + + return topicOffsets + }) + + // Execute all requests, merge and normalize the responses + const responses = await Promise.all(requests) + const partitionsPerTopic = responses.flat().reduce(mergeTopics, {}) + + return keys(partitionsPerTopic).map(topic => ({ + topic, + partitions: partitionsPerTopic[topic].map(({ partition, offset }) => ({ + partition, + offset, + })), + })) + } + + /** + * Retrieve the object mapping for committed offsets for a single consumer group + * @param {object} options + * @param {string} options.groupId + * @returns {Object} + */ + committedOffsets({ groupId }) { + if (!this.committedOffsetsByGroup.has(groupId)) { + this.committedOffsetsByGroup.set(groupId, {}) + } + + return this.committedOffsetsByGroup.get(groupId) + } + + /** + * Mark offset as committed for a single consumer group's topic-partition + * @param {object} options + * @param {string} options.groupId + * @param {string} options.topic + * @param {string|number} options.partition + * @param {string} options.offset + */ + markOffsetAsCommitted({ groupId, topic, partition, offset }) { + const committedOffsets = this.committedOffsets({ groupId }) + + committedOffsets[topic] = committedOffsets[topic] || {} + committedOffsets[topic][partition] = offset + } +} diff --git a/node_modules/kafkajs/src/constants.js b/node_modules/kafkajs/src/constants.js new file mode 100644 index 0000000..bf1e600 --- /dev/null +++ b/node_modules/kafkajs/src/constants.js @@ -0,0 +1,9 @@ +const EARLIEST_OFFSET = -2 +const LATEST_OFFSET = -1 +const INT_32_MAX_VALUE = Math.pow(2, 31) - 1 + +module.exports = { + EARLIEST_OFFSET, + LATEST_OFFSET, + INT_32_MAX_VALUE, +} diff --git a/node_modules/kafkajs/src/consumer/assignerProtocol.js b/node_modules/kafkajs/src/consumer/assignerProtocol.js new file mode 100644 index 0000000..cb064a5 --- /dev/null +++ b/node_modules/kafkajs/src/consumer/assignerProtocol.js @@ -0,0 +1,87 @@ +const Encoder = require('../protocol/encoder') +const Decoder = require('../protocol/decoder') + +const MemberMetadata = { + /** + * @param {Object} metadata + * @param {number} metadata.version + * @param {Array} metadata.topics + * @param {Buffer} [metadata.userData=Buffer.alloc(0)] + * + * @returns Buffer + */ + encode({ version, topics, userData = Buffer.alloc(0) }) { + return new Encoder() + .writeInt16(version) + .writeArray(topics) + .writeBytes(userData).buffer + }, + + /** + * @param {Buffer} buffer + * @returns {Object} + */ + decode(buffer) { + const decoder = new Decoder(buffer) + return { + version: decoder.readInt16(), + topics: decoder.readArray(d => d.readString()), + userData: decoder.readBytes(), + } + }, +} + +const MemberAssignment = { + /** + * @param {object} options + * @param {number} options.version + * @param {Object} options.assignment, example: + * { + * 'topic-A': [0, 2, 4, 6], + * 'topic-B': [0, 2], + * } + * @param {Buffer} [options.userData=Buffer.alloc(0)] + * + * @returns Buffer + */ + encode({ version, assignment, userData = Buffer.alloc(0) }) { + return new Encoder() + .writeInt16(version) + .writeArray( + Object.keys(assignment).map(topic => + new Encoder().writeString(topic).writeArray(assignment[topic]) + ) + ) + .writeBytes(userData).buffer + }, + + /** + * @param {Buffer} buffer + * @returns {Object|null} + */ + decode(buffer) { + const decoder = new Decoder(buffer) + const decodePartitions = d => d.readInt32() + const decodeAssignment = d => ({ + topic: d.readString(), + partitions: d.readArray(decodePartitions), + }) + const indexAssignment = (obj, { topic, partitions }) => + Object.assign(obj, { [topic]: partitions }) + + if (!decoder.canReadInt16()) { + return null + } + + return { + version: decoder.readInt16(), + assignment: decoder.readArray(decodeAssignment).reduce(indexAssignment, {}), + userData: decoder.readBytes(), + } + }, +} + +module.exports = { + MemberMetadata, + MemberAssignment, +} diff --git a/node_modules/kafkajs/src/consumer/assigners/index.js b/node_modules/kafkajs/src/consumer/assigners/index.js new file mode 100644 index 0000000..1709106 --- /dev/null +++ b/node_modules/kafkajs/src/consumer/assigners/index.js @@ -0,0 +1,5 @@ +const roundRobin = require('./roundRobinAssigner') + +module.exports = { + roundRobin, +} diff --git a/node_modules/kafkajs/src/consumer/assigners/roundRobinAssigner/index.js b/node_modules/kafkajs/src/consumer/assigners/roundRobinAssigner/index.js new file mode 100644 index 0000000..254c414 --- /dev/null +++ b/node_modules/kafkajs/src/consumer/assigners/roundRobinAssigner/index.js @@ -0,0 +1,81 @@ +const { MemberMetadata, MemberAssignment } = require('../../assignerProtocol') + +/** + * RoundRobinAssigner + * @type {import('types').PartitionAssigner} + */ +module.exports = ({ cluster }) => ({ + name: 'RoundRobinAssigner', + version: 0, + + /** + * Assign the topics to the provided members. + * + * The members array contains information about each member, `memberMetadata` is the result of the + * `protocol` operation. + * + * @param {object} group + * @param {import('types').GroupMember[]} group.members array of members, e.g: + [{ memberId: 'test-5f93f5a3', memberMetadata: Buffer }] + * @param {string[]} group.topics + * @returns {Promise} object partitions per topic per member, e.g: + * [ + * { + * memberId: 'test-5f93f5a3', + * memberAssignment: { + * 'topic-A': [0, 2, 4, 6], + * 'topic-B': [1], + * }, + * }, + * { + * memberId: 'test-3d3d5341', + * memberAssignment: { + * 'topic-A': [1, 3, 5], + * 'topic-B': [0, 2], + * }, + * } + * ] + */ + async assign({ members, topics }) { + const membersCount = members.length + const sortedMembers = members.map(({ memberId }) => memberId).sort() + const assignment = {} + + const topicsPartitions = topics.flatMap(topic => { + const partitionMetadata = cluster.findTopicPartitionMetadata(topic) + return partitionMetadata.map(m => ({ topic: topic, partitionId: m.partitionId })) + }) + + topicsPartitions.forEach((topicPartition, i) => { + const assignee = sortedMembers[i % membersCount] + + if (!assignment[assignee]) { + assignment[assignee] = Object.create(null) + } + + if (!assignment[assignee][topicPartition.topic]) { + assignment[assignee][topicPartition.topic] = [] + } + + assignment[assignee][topicPartition.topic].push(topicPartition.partitionId) + }) + + return Object.keys(assignment).map(memberId => ({ + memberId, + memberAssignment: MemberAssignment.encode({ + version: this.version, + assignment: assignment[memberId], + }), + })) + }, + + protocol({ topics }) { + return { + name: this.name, + metadata: MemberMetadata.encode({ + version: this.version, + topics, + }), + } + }, +}) diff --git a/node_modules/kafkajs/src/consumer/batch.js b/node_modules/kafkajs/src/consumer/batch.js new file mode 100644 index 0000000..8b1965a --- /dev/null +++ b/node_modules/kafkajs/src/consumer/batch.js @@ -0,0 +1,112 @@ +const Long = require('../utils/long') +const filterAbortedMessages = require('./filterAbortedMessages') + +/** + * A batch collects messages returned from a single fetch call. + * + * A batch could contain _multiple_ Kafka RecordBatches. + */ +module.exports = class Batch { + constructor(topic, fetchedOffset, partitionData) { + this.fetchedOffset = fetchedOffset + const longFetchedOffset = Long.fromValue(this.fetchedOffset) + const { abortedTransactions, messages } = partitionData + + this.topic = topic + this.partition = partitionData.partition + this.highWatermark = partitionData.highWatermark + + this.rawMessages = messages + // Apparently fetch can return different offsets than the target offset provided to the fetch API. + // Discard messages that are not in the requested offset + // https://github.com/apache/kafka/blob/bf237fa7c576bd141d78fdea9f17f65ea269c290/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java#L912 + this.messagesWithinOffset = this.rawMessages.filter(message => + Long.fromValue(message.offset).gte(longFetchedOffset) + ) + + // 1. Don't expose aborted messages + // 2. Don't expose control records + // @see https://kafka.apache.org/documentation/#controlbatch + this.messages = filterAbortedMessages({ + messages: this.messagesWithinOffset, + abortedTransactions, + }).filter(message => !message.isControlRecord) + } + + isEmpty() { + return this.messages.length === 0 + } + + isEmptyIncludingFiltered() { + return this.messagesWithinOffset.length === 0 + } + + /** + * If the batch contained raw messages (i.e was not truly empty) but all messages were filtered out due to + * log compaction, control records or other reasons + */ + isEmptyDueToFiltering() { + return this.isEmpty() && this.rawMessages.length > 0 + } + + isEmptyControlRecord() { + return ( + this.isEmpty() && this.messagesWithinOffset.some(({ isControlRecord }) => isControlRecord) + ) + } + + /** + * With compressed messages, it's possible for the returned messages to have offsets smaller than the starting offset. + * These messages will be filtered out (i.e. they are not even included in this.messagesWithinOffset) + * If these are the only messages, the batch will appear as an empty batch. + * + * isEmpty() and isEmptyIncludingFiltered() will always return true if the batch is empty, + * but this method will only return true if the batch is empty due to log compacted messages. + * + * @returns boolean True if the batch is empty, because of log compacted messages in the partition. + */ + isEmptyDueToLogCompactedMessages() { + const hasMessages = this.rawMessages.length > 0 + return hasMessages && this.isEmptyIncludingFiltered() + } + + firstOffset() { + return this.isEmptyIncludingFiltered() ? null : this.messagesWithinOffset[0].offset + } + + lastOffset() { + if (this.isEmptyDueToLogCompactedMessages()) { + return this.fetchedOffset + } + + if (this.isEmptyIncludingFiltered()) { + return Long.fromValue(this.highWatermark) + .add(-1) + .toString() + } + + return this.messagesWithinOffset[this.messagesWithinOffset.length - 1].offset + } + + /** + * Returns the lag based on the last offset in the batch (also known as "high") + */ + offsetLag() { + const lastOffsetOfPartition = Long.fromValue(this.highWatermark).add(-1) + const lastConsumedOffset = Long.fromValue(this.lastOffset()) + return lastOffsetOfPartition.add(lastConsumedOffset.multiply(-1)).toString() + } + + /** + * Returns the lag based on the first offset in the batch + */ + offsetLagLow() { + if (this.isEmptyIncludingFiltered()) { + return '0' + } + + const lastOffsetOfPartition = Long.fromValue(this.highWatermark).add(-1) + const firstConsumedOffset = Long.fromValue(this.firstOffset()) + return lastOffsetOfPartition.add(firstConsumedOffset.multiply(-1)).toString() + } +} diff --git a/node_modules/kafkajs/src/consumer/consumerGroup.js b/node_modules/kafkajs/src/consumer/consumerGroup.js new file mode 100644 index 0000000..442312c --- /dev/null +++ b/node_modules/kafkajs/src/consumer/consumerGroup.js @@ -0,0 +1,759 @@ +const sleep = require('../utils/sleep') +const websiteUrl = require('../utils/websiteUrl') +const arrayDiff = require('../utils/arrayDiff') +const createRetry = require('../retry') +const sharedPromiseTo = require('../utils/sharedPromiseTo') + +const OffsetManager = require('./offsetManager') +const Batch = require('./batch') +const SeekOffsets = require('./seekOffsets') +const SubscriptionState = require('./subscriptionState') +const { + events: { GROUP_JOIN, HEARTBEAT, CONNECT, RECEIVED_UNSUBSCRIBED_TOPICS }, +} = require('./instrumentationEvents') +const { MemberAssignment } = require('./assignerProtocol') +const { + KafkaJSError, + KafkaJSNonRetriableError, + KafkaJSStaleTopicMetadataAssignment, + isRebalancing, +} = require('../errors') + +const { keys } = Object + +const STALE_METADATA_ERRORS = [ + 'LEADER_NOT_AVAILABLE', + // Fetch before v9 uses NOT_LEADER_FOR_PARTITION + 'NOT_LEADER_FOR_PARTITION', + // Fetch after v9 uses {FENCED,UNKNOWN}_LEADER_EPOCH + 'FENCED_LEADER_EPOCH', + 'UNKNOWN_LEADER_EPOCH', + 'UNKNOWN_TOPIC_OR_PARTITION', +] + +const PRIVATE = { + JOIN: Symbol('private:ConsumerGroup:join'), + SYNC: Symbol('private:ConsumerGroup:sync'), + SHARED_HEARTBEAT: Symbol('private:ConsumerGroup:sharedHeartbeat'), +} + +module.exports = class ConsumerGroup { + /** + * @param {object} options + * @param {import('../../types').RetryOptions} options.retry + * @param {import('../../types').Cluster} options.cluster + * @param {string} options.groupId + * @param {string[]} options.topics + * @param {Record} options.topicConfigurations + * @param {import('../../types').Logger} options.logger + * @param {import('../instrumentation/emitter')} options.instrumentationEmitter + * @param {import('../../types').Assigner[]} options.assigners + * @param {number} options.sessionTimeout + * @param {number} options.rebalanceTimeout + * @param {number} options.maxBytesPerPartition + * @param {number} options.minBytes + * @param {number} options.maxBytes + * @param {number} options.maxWaitTimeInMs + * @param {boolean} options.autoCommit + * @param {number} options.autoCommitInterval + * @param {number} options.autoCommitThreshold + * @param {number} options.isolationLevel + * @param {string} options.rackId + * @param {number} options.metadataMaxAge + */ + constructor({ + retry, + cluster, + groupId, + topics, + topicConfigurations, + logger, + instrumentationEmitter, + assigners, + sessionTimeout, + rebalanceTimeout, + maxBytesPerPartition, + minBytes, + maxBytes, + maxWaitTimeInMs, + autoCommit, + autoCommitInterval, + autoCommitThreshold, + isolationLevel, + rackId, + metadataMaxAge, + }) { + /** @type {import("../../types").Cluster} */ + this.cluster = cluster + this.groupId = groupId + this.topics = topics + this.topicsSubscribed = topics + this.topicConfigurations = topicConfigurations + this.logger = logger.namespace('ConsumerGroup') + this.instrumentationEmitter = instrumentationEmitter + this.retrier = createRetry(Object.assign({}, retry)) + this.assigners = assigners + this.sessionTimeout = sessionTimeout + this.rebalanceTimeout = rebalanceTimeout + this.maxBytesPerPartition = maxBytesPerPartition + this.minBytes = minBytes + this.maxBytes = maxBytes + this.maxWaitTime = maxWaitTimeInMs + this.autoCommit = autoCommit + this.autoCommitInterval = autoCommitInterval + this.autoCommitThreshold = autoCommitThreshold + this.isolationLevel = isolationLevel + this.rackId = rackId + this.metadataMaxAge = metadataMaxAge + + this.seekOffset = new SeekOffsets() + this.coordinator = null + this.generationId = null + this.leaderId = null + this.memberId = null + this.members = null + this.groupProtocol = null + + this.partitionsPerSubscribedTopic = null + /** + * Preferred read replica per topic and partition + * + * Each of the partitions tracks the preferred read replica (`nodeId`) and a timestamp + * until when that preference is valid. + * + * @type {{[topicName: string]: {[partition: number]: {nodeId: number, expireAt: number}}}} + */ + this.preferredReadReplicasPerTopicPartition = {} + this.offsetManager = null + this.subscriptionState = new SubscriptionState() + + this.lastRequest = Date.now() + + this[PRIVATE.SHARED_HEARTBEAT] = sharedPromiseTo(async ({ interval }) => { + const { groupId, generationId, memberId } = this + const now = Date.now() + + if (memberId && now >= this.lastRequest + interval) { + const payload = { + groupId, + memberId, + groupGenerationId: generationId, + } + + await this.coordinator.heartbeat(payload) + this.instrumentationEmitter.emit(HEARTBEAT, payload) + this.lastRequest = Date.now() + } + }) + } + + isLeader() { + return this.leaderId && this.memberId === this.leaderId + } + + getNodeIds() { + return this.cluster.getNodeIds() + } + + async connect() { + await this.cluster.connect() + this.instrumentationEmitter.emit(CONNECT) + await this.cluster.refreshMetadataIfNecessary() + } + + async [PRIVATE.JOIN]() { + const { groupId, sessionTimeout, rebalanceTimeout } = this + + this.coordinator = await this.cluster.findGroupCoordinator({ groupId }) + + const groupData = await this.coordinator.joinGroup({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId: this.memberId || '', + groupProtocols: this.assigners.map(assigner => + assigner.protocol({ + topics: this.topicsSubscribed, + }) + ), + }) + + this.generationId = groupData.generationId + this.leaderId = groupData.leaderId + this.memberId = groupData.memberId + this.members = groupData.members + this.groupProtocol = groupData.groupProtocol + } + + async leave() { + const { groupId, memberId } = this + if (memberId) { + await this.coordinator.leaveGroup({ groupId, memberId }) + this.memberId = null + } + } + + async [PRIVATE.SYNC]() { + let assignment = [] + const { + groupId, + generationId, + memberId, + members, + groupProtocol, + topics, + topicsSubscribed, + coordinator, + } = this + + if (this.isLeader()) { + this.logger.debug('Chosen as group leader', { groupId, generationId, memberId, topics }) + const assigner = this.assigners.find(({ name }) => name === groupProtocol) + + if (!assigner) { + throw new KafkaJSNonRetriableError( + `Unsupported partition assigner "${groupProtocol}", the assigner wasn't found in the assigners list` + ) + } + + await this.cluster.refreshMetadata() + assignment = await assigner.assign({ members, topics: topicsSubscribed }) + + this.logger.debug('Group assignment', { + groupId, + generationId, + groupProtocol, + assignment, + topics: topicsSubscribed, + }) + } + + // Keep track of the partitions for the subscribed topics + this.partitionsPerSubscribedTopic = this.generatePartitionsPerSubscribedTopic() + const { memberAssignment } = await this.coordinator.syncGroup({ + groupId, + generationId, + memberId, + groupAssignment: assignment, + }) + + const decodedMemberAssignment = MemberAssignment.decode(memberAssignment) + const decodedAssignment = + decodedMemberAssignment != null ? decodedMemberAssignment.assignment : {} + + this.logger.debug('Received assignment', { + groupId, + generationId, + memberId, + memberAssignment: decodedAssignment, + }) + + const assignedTopics = keys(decodedAssignment) + const topicsNotSubscribed = arrayDiff(assignedTopics, topicsSubscribed) + + if (topicsNotSubscribed.length > 0) { + const payload = { + groupId, + generationId, + memberId, + assignedTopics, + topicsSubscribed, + topicsNotSubscribed, + } + + this.instrumentationEmitter.emit(RECEIVED_UNSUBSCRIBED_TOPICS, payload) + this.logger.warn('Consumer group received unsubscribed topics', { + ...payload, + helpUrl: websiteUrl( + 'docs/faq', + 'why-am-i-receiving-messages-for-topics-i-m-not-subscribed-to' + ), + }) + } + + // Remove unsubscribed topics from the list + const safeAssignment = arrayDiff(assignedTopics, topicsNotSubscribed) + const currentMemberAssignment = safeAssignment.map(topic => ({ + topic, + partitions: decodedAssignment[topic], + })) + + // Check if the consumer is aware of all assigned partitions + for (const assignment of currentMemberAssignment) { + const { topic, partitions: assignedPartitions } = assignment + const knownPartitions = this.partitionsPerSubscribedTopic.get(topic) + const isAwareOfAllAssignedPartitions = assignedPartitions.every(partition => + knownPartitions.includes(partition) + ) + + if (!isAwareOfAllAssignedPartitions) { + this.logger.warn('Consumer is not aware of all assigned partitions, refreshing metadata', { + groupId, + generationId, + memberId, + topic, + knownPartitions, + assignedPartitions, + }) + + // If the consumer is not aware of all assigned partitions, refresh metadata + // and update the list of partitions per subscribed topic. It's enough to perform + // this operation once since refresh metadata will update metadata for all topics + await this.cluster.refreshMetadata() + this.partitionsPerSubscribedTopic = this.generatePartitionsPerSubscribedTopic() + break + } + } + + this.topics = currentMemberAssignment.map(({ topic }) => topic) + this.subscriptionState.assign(currentMemberAssignment) + this.offsetManager = new OffsetManager({ + cluster: this.cluster, + topicConfigurations: this.topicConfigurations, + instrumentationEmitter: this.instrumentationEmitter, + memberAssignment: currentMemberAssignment.reduce( + (partitionsByTopic, { topic, partitions }) => ({ + ...partitionsByTopic, + [topic]: partitions, + }), + {} + ), + autoCommit: this.autoCommit, + autoCommitInterval: this.autoCommitInterval, + autoCommitThreshold: this.autoCommitThreshold, + coordinator, + groupId, + generationId, + memberId, + }) + } + + joinAndSync() { + const startJoin = Date.now() + return this.retrier(async bail => { + try { + await this[PRIVATE.JOIN]() + await this[PRIVATE.SYNC]() + + const memberAssignment = this.assigned().reduce( + (result, { topic, partitions }) => ({ ...result, [topic]: partitions }), + {} + ) + + const payload = { + groupId: this.groupId, + memberId: this.memberId, + leaderId: this.leaderId, + isLeader: this.isLeader(), + memberAssignment, + groupProtocol: this.groupProtocol, + duration: Date.now() - startJoin, + } + + this.instrumentationEmitter.emit(GROUP_JOIN, payload) + this.logger.info('Consumer has joined the group', payload) + } catch (e) { + if (isRebalancing(e)) { + // Rebalance in progress isn't a retriable protocol error since the consumer + // has to go through find coordinator and join again before it can + // actually retry the operation. We wrap the original error in a retriable error + // here instead in order to restart the join + sync sequence using the retrier. + throw new KafkaJSError(e) + } + + if (e.type === 'UNKNOWN_MEMBER_ID') { + this.memberId = null + throw new KafkaJSError(e) + } + + bail(e) + } + }) + } + + /** + * @param {import("../../types").TopicPartition} topicPartition + */ + resetOffset({ topic, partition }) { + this.offsetManager.resetOffset({ topic, partition }) + } + + /** + * @param {import("../../types").TopicPartitionOffset} topicPartitionOffset + */ + resolveOffset({ topic, partition, offset }) { + this.offsetManager.resolveOffset({ topic, partition, offset }) + } + + /** + * Update the consumer offset for the given topic/partition. This will be used + * on the next fetch. If this API is invoked for the same topic/partition more + * than once, the latest offset will be used on the next fetch. + * + * @param {import("../../types").TopicPartitionOffset} topicPartitionOffset + */ + seek({ topic, partition, offset }) { + this.seekOffset.set(topic, partition, offset) + } + + pause(topicPartitions) { + this.logger.info(`Pausing fetching from ${topicPartitions.length} topics`, { + topicPartitions, + }) + this.subscriptionState.pause(topicPartitions) + } + + resume(topicPartitions) { + this.logger.info(`Resuming fetching from ${topicPartitions.length} topics`, { + topicPartitions, + }) + this.subscriptionState.resume(topicPartitions) + } + + assigned() { + return this.subscriptionState.assigned() + } + + paused() { + return this.subscriptionState.paused() + } + + /** + * @param {string} topic + * @param {string} partition + * @returns {boolean} whether the specified topic-partition are paused or not + */ + isPaused(topic, partition) { + return this.subscriptionState.isPaused(topic, partition) + } + + async commitOffsetsIfNecessary() { + await this.offsetManager.commitOffsetsIfNecessary() + } + + async commitOffsets(offsets) { + await this.offsetManager.commitOffsets(offsets) + } + + uncommittedOffsets() { + return this.offsetManager.uncommittedOffsets() + } + + async heartbeat({ interval }) { + return this[PRIVATE.SHARED_HEARTBEAT]({ interval }) + } + + async fetch(nodeId) { + try { + await this.cluster.refreshMetadataIfNecessary() + this.checkForStaleAssignment() + + let topicPartitions = this.subscriptionState.assigned() + topicPartitions = this.filterPartitionsByNode(nodeId, topicPartitions) + + await this.seekOffsets(topicPartitions) + + const committedOffsets = this.offsetManager.committedOffsets() + const activeTopicPartitions = this.getActiveTopicPartitions() + + const requests = topicPartitions + .map(({ topic, partitions }) => ({ + topic, + partitions: partitions + .filter( + partition => + /** + * When recovering from OffsetOutOfRange, each partition can recover + * concurrently, which invalidates resolved and committed offsets as part + * of the recovery mechanism (see OffsetManager.clearOffsets). In concurrent + * scenarios this can initiate a new fetch with invalid offsets. + * + * This was further highlighted by https://github.com/tulios/kafkajs/pull/570, + * which increased concurrency, making this more likely to happen. + * + * This is solved by only making requests for partitions with initialized offsets. + * + * See the following pull request which explains the context of the problem: + * @issue https://github.com/tulios/kafkajs/pull/578 + */ + committedOffsets[topic][partition] != null && + activeTopicPartitions[topic].has(partition) + ) + .map(partition => ({ + partition, + fetchOffset: this.offsetManager.nextOffset(topic, partition).toString(), + maxBytes: this.maxBytesPerPartition, + })), + })) + .filter(({ partitions }) => partitions.length) + + if (!requests.length) { + await sleep(this.maxWaitTime) + return [] + } + + const broker = await this.cluster.findBroker({ nodeId }) + + const { responses } = await broker.fetch({ + maxWaitTime: this.maxWaitTime, + minBytes: this.minBytes, + maxBytes: this.maxBytes, + isolationLevel: this.isolationLevel, + topics: requests, + rackId: this.rackId, + }) + + return responses.flatMap(({ topicName, partitions }) => { + const topicRequestData = requests.find(({ topic }) => topic === topicName) + + let preferredReadReplicas = this.preferredReadReplicasPerTopicPartition[topicName] + if (!preferredReadReplicas) { + this.preferredReadReplicasPerTopicPartition[topicName] = preferredReadReplicas = {} + } + + return partitions + .filter( + ({ partition }) => + !this.seekOffset.has(topicName, partition) && + !this.subscriptionState.isPaused(topicName, partition) + ) + .map(partitionData => { + const { partition, preferredReadReplica } = partitionData + + if (preferredReadReplica != null && preferredReadReplica !== -1) { + const { nodeId: currentPreferredReadReplica } = preferredReadReplicas[partition] || {} + if (currentPreferredReadReplica !== preferredReadReplica) { + this.logger.info(`Preferred read replica is now ${preferredReadReplica}`, { + groupId: this.groupId, + memberId: this.memberId, + topic: topicName, + partition, + }) + } + preferredReadReplicas[partition] = { + nodeId: preferredReadReplica, + expireAt: Date.now() + this.metadataMaxAge, + } + } + + const partitionRequestData = topicRequestData.partitions.find( + ({ partition }) => partition === partitionData.partition + ) + + const fetchedOffset = partitionRequestData.fetchOffset + return new Batch(topicName, fetchedOffset, partitionData) + }) + }) + } catch (e) { + await this.recoverFromFetch(e) + return [] + } + } + + async recoverFromFetch(e) { + if (STALE_METADATA_ERRORS.includes(e.type) || e.name === 'KafkaJSTopicMetadataNotLoaded') { + this.logger.debug('Stale cluster metadata, refreshing...', { + groupId: this.groupId, + memberId: this.memberId, + error: e.message, + }) + + await this.cluster.refreshMetadata() + await this.joinAndSync() + return + } + + if (e.name === 'KafkaJSStaleTopicMetadataAssignment') { + this.logger.warn(`${e.message}, resync group`, { + groupId: this.groupId, + memberId: this.memberId, + topic: e.topic, + unknownPartitions: e.unknownPartitions, + }) + + await this.joinAndSync() + return + } + + if (e.name === 'KafkaJSOffsetOutOfRange') { + await this.recoverFromOffsetOutOfRange(e) + return + } + + if (e.name === 'KafkaJSConnectionClosedError') { + this.cluster.removeBroker({ host: e.host, port: e.port }) + return + } + + if (e.name === 'KafkaJSBrokerNotFound' || e.name === 'KafkaJSConnectionClosedError') { + this.logger.debug(`${e.message}, refreshing metadata and retrying...`) + await this.cluster.refreshMetadata() + return + } + + throw e + } + + async recoverFromOffsetOutOfRange(e) { + // If we are fetching from a follower try with the leader before resetting offsets + const preferredReadReplicas = this.preferredReadReplicasPerTopicPartition[e.topic] + if (preferredReadReplicas && typeof preferredReadReplicas[e.partition] === 'number') { + this.logger.info('Offset out of range while fetching from follower, retrying with leader', { + topic: e.topic, + partition: e.partition, + groupId: this.groupId, + memberId: this.memberId, + }) + delete preferredReadReplicas[e.partition] + } else { + this.logger.error('Offset out of range, resetting to default offset', { + topic: e.topic, + partition: e.partition, + groupId: this.groupId, + memberId: this.memberId, + }) + + await this.offsetManager.setDefaultOffset({ + topic: e.topic, + partition: e.partition, + }) + } + } + + generatePartitionsPerSubscribedTopic() { + const map = new Map() + + for (const topic of this.topicsSubscribed) { + const partitions = this.cluster + .findTopicPartitionMetadata(topic) + .map(m => m.partitionId) + .sort() + + map.set(topic, partitions) + } + + return map + } + + checkForStaleAssignment() { + if (!this.partitionsPerSubscribedTopic) { + return + } + + const newPartitionsPerSubscribedTopic = this.generatePartitionsPerSubscribedTopic() + + for (const [topic, partitions] of newPartitionsPerSubscribedTopic) { + const diff = arrayDiff(partitions, this.partitionsPerSubscribedTopic.get(topic)) + + if (diff.length > 0) { + throw new KafkaJSStaleTopicMetadataAssignment('Topic has been updated', { + topic, + unknownPartitions: diff, + }) + } + } + } + + async seekOffsets(topicPartitions) { + for (const { topic, partitions } of topicPartitions) { + for (const partition of partitions) { + const seekEntry = this.seekOffset.pop(topic, partition) + if (!seekEntry) { + continue + } + + this.logger.debug('Seek offset', { + groupId: this.groupId, + memberId: this.memberId, + seek: seekEntry, + }) + await this.offsetManager.seek(seekEntry) + } + } + + await this.offsetManager.resolveOffsets() + } + + hasSeekOffset({ topic, partition }) { + return this.seekOffset.has(topic, partition) + } + + /** + * For each of the partitions find the best nodeId to read it from + * + * @param {string} topic + * @param {number[]} partitions + * @returns {{[nodeId: number]: number[]}} per-node assignment of partitions + * @see Cluster~findLeaderForPartitions + */ + // Invariant: The resulting object has each partition referenced exactly once + findReadReplicaForPartitions(topic, partitions) { + const partitionMetadata = this.cluster.findTopicPartitionMetadata(topic) + const preferredReadReplicas = this.preferredReadReplicasPerTopicPartition[topic] + return partitions.reduce((result, id) => { + const partitionId = parseInt(id, 10) + const metadata = partitionMetadata.find(p => p.partitionId === partitionId) + if (!metadata) { + return result + } + + if (metadata.leader == null) { + throw new KafkaJSError('Invalid partition metadata', { topic, partitionId, metadata }) + } + + // Pick the preferred replica if there is one, and it isn't known to be offline, otherwise the leader. + let nodeId = metadata.leader + if (preferredReadReplicas) { + const { nodeId: preferredReadReplica, expireAt } = preferredReadReplicas[partitionId] || {} + if (Date.now() >= expireAt) { + this.logger.debug('Preferred read replica information has expired, using leader', { + topic, + partitionId, + groupId: this.groupId, + memberId: this.memberId, + preferredReadReplica, + leader: metadata.leader, + }) + // Drop the entry + delete preferredReadReplicas[partitionId] + } else if (preferredReadReplica != null) { + // Valid entry, check whether it is not offline + // Note that we don't delete the preference here, and rather hope that eventually that replica comes online again + const offlineReplicas = metadata.offlineReplicas + if (Array.isArray(offlineReplicas) && offlineReplicas.includes(nodeId)) { + this.logger.debug('Preferred read replica is offline, using leader', { + topic, + partitionId, + groupId: this.groupId, + memberId: this.memberId, + preferredReadReplica, + leader: metadata.leader, + }) + } else { + nodeId = preferredReadReplica + } + } + } + const current = result[nodeId] || [] + return { ...result, [nodeId]: [...current, partitionId] } + }, {}) + } + + filterPartitionsByNode(nodeId, topicPartitions) { + return topicPartitions.map(({ topic, partitions }) => ({ + topic, + partitions: this.findReadReplicaForPartitions(topic, partitions)[nodeId] || [], + })) + } + + getActiveTopicPartitions() { + const activeSubscriptionState = this.subscriptionState.active() + + const activeTopicPartitions = {} + activeSubscriptionState.forEach(({ topic, partitions }) => { + activeTopicPartitions[topic] = new Set(partitions) + }) + + return activeTopicPartitions + } +} diff --git a/node_modules/kafkajs/src/consumer/fetchManager.js b/node_modules/kafkajs/src/consumer/fetchManager.js new file mode 100644 index 0000000..6b5cf65 --- /dev/null +++ b/node_modules/kafkajs/src/consumer/fetchManager.js @@ -0,0 +1,99 @@ +const seq = require('../utils/seq') +const createFetcher = require('./fetcher') +const createWorker = require('./worker') +const createWorkerQueue = require('./workerQueue') +const { KafkaJSFetcherRebalanceError, KafkaJSNoBrokerAvailableError } = require('../errors') + +/** @typedef {ReturnType} FetchManager */ + +/** + * @param {object} options + * @param {import('../../types').Logger} options.logger + * @param {() => number[]} options.getNodeIds + * @param {(nodeId: number) => Promise} options.fetch + * @param {import('./worker').Handler} options.handler + * @param {number} [options.concurrency] + * @template T + */ +const createFetchManager = ({ + logger: rootLogger, + getNodeIds, + fetch, + handler, + concurrency = 1, +}) => { + const logger = rootLogger.namespace('FetchManager') + const workers = seq(concurrency, workerId => createWorker({ handler, workerId })) + const workerQueue = createWorkerQueue({ workers }) + + let fetchers = [] + + const getFetchers = () => fetchers + + const createFetchers = () => { + const nodeIds = getNodeIds() + const partitionAssignments = new Map() + + if (nodeIds.length === 0) { + throw new KafkaJSNoBrokerAvailableError() + } + + const validateShouldRebalance = () => { + const current = getNodeIds() + const hasChanged = + nodeIds.length !== current.length || nodeIds.some(nodeId => !current.includes(nodeId)) + if (hasChanged && current.length !== 0) { + throw new KafkaJSFetcherRebalanceError() + } + } + + const fetchers = nodeIds.map(nodeId => + createFetcher({ + nodeId, + workerQueue, + partitionAssignments, + fetch: async nodeId => { + validateShouldRebalance() + return fetch(nodeId) + }, + logger, + }) + ) + + logger.debug(`Created ${fetchers.length} fetchers`, { nodeIds, concurrency }) + return fetchers + } + + const start = async () => { + logger.debug('Starting...') + + while (true) { + fetchers = createFetchers() + + try { + await Promise.all(fetchers.map(fetcher => fetcher.start())) + } catch (error) { + await stop() + + if (error instanceof KafkaJSFetcherRebalanceError) { + logger.debug('Rebalancing fetchers...') + continue + } + + throw error + } + + break + } + } + + const stop = async () => { + logger.debug('Stopping fetchers...') + await Promise.all(fetchers.map(fetcher => fetcher.stop())) + logger.debug('Stopped fetchers') + } + + return { start, stop, getFetchers } +} + +module.exports = createFetchManager diff --git a/node_modules/kafkajs/src/consumer/fetcher.js b/node_modules/kafkajs/src/consumer/fetcher.js new file mode 100644 index 0000000..fcea2ba --- /dev/null +++ b/node_modules/kafkajs/src/consumer/fetcher.js @@ -0,0 +1,86 @@ +const EventEmitter = require('events') + +/** + * Fetches data from all assigned nodes, waits for workerQueue to drain and repeats. + * + * @param {object} options + * @param {number} options.nodeId + * @param {import('./workerQueue').WorkerQueue} options.workerQueue + * @param {Map} options.partitionAssignments + * @param {(nodeId: number) => Promise} options.fetch + * @param {import('../../types').Logger} options.logger + * @template T + */ +const createFetcher = ({ + nodeId, + workerQueue, + partitionAssignments, + fetch, + logger: rootLogger, +}) => { + const logger = rootLogger.namespace(`Fetcher ${nodeId}`) + const emitter = new EventEmitter() + let isRunning = false + + const getWorkerQueue = () => workerQueue + const assignmentKey = ({ topic, partition }) => `${topic}|${partition}` + const getAssignedFetcher = batch => partitionAssignments.get(assignmentKey(batch)) + const assignTopicPartition = batch => partitionAssignments.set(assignmentKey(batch), nodeId) + const unassignTopicPartition = batch => partitionAssignments.delete(assignmentKey(batch)) + const filterUnassignedBatches = batches => + batches.filter(batch => { + const assignedFetcher = getAssignedFetcher(batch) + if (assignedFetcher != null && assignedFetcher !== nodeId) { + logger.info( + 'Filtering out batch due to partition already being processed by another fetcher', + { + topic: batch.topic, + partition: batch.partition, + assignedFetcher: assignedFetcher, + fetcher: nodeId, + } + ) + return false + } + + return true + }) + + const start = async () => { + if (isRunning) return + isRunning = true + + while (isRunning) { + try { + const batches = await fetch(nodeId) + if (isRunning) { + const availableBatches = filterUnassignedBatches(batches) + + if (availableBatches.length > 0) { + availableBatches.forEach(assignTopicPartition) + try { + await workerQueue.push(...availableBatches) + } finally { + availableBatches.forEach(unassignTopicPartition) + } + } + } + } catch (error) { + isRunning = false + emitter.emit('end') + throw error + } + } + emitter.emit('end') + } + + const stop = async () => { + if (!isRunning) return + isRunning = false + await new Promise(resolve => emitter.once('end', () => resolve())) + } + + return { start, stop, getWorkerQueue } +} + +module.exports = createFetcher diff --git a/node_modules/kafkajs/src/consumer/filterAbortedMessages.js b/node_modules/kafkajs/src/consumer/filterAbortedMessages.js new file mode 100644 index 0000000..6f3b794 --- /dev/null +++ b/node_modules/kafkajs/src/consumer/filterAbortedMessages.js @@ -0,0 +1,64 @@ +const Long = require('../utils/long') +const ABORTED_MESSAGE_KEY = Buffer.from([0, 0, 0, 0]) + +const isAbortMarker = ({ key }) => { + // Handle null/undefined keys. + if (!key) return false + // Cast key to buffer defensively + return Buffer.from(key).equals(ABORTED_MESSAGE_KEY) +} + +/** + * Remove messages marked as aborted according to the aborted transactions list. + * + * Start of an aborted transaction is determined by message offset. + * End of an aborted transaction is determined by control messages. + * @param {Message[]} messages + * @param {Transaction[]} [abortedTransactions] + * @returns {Message[]} Messages which did not participate in an aborted transaction + * + * @typedef {object} Message + * @param {Buffer} key + * @param {lastOffset} key Int64 + * @param {RecordBatch} batchContext + * + * @typedef {object} Transaction + * @param {string} firstOffset Int64 + * @param {string} producerId Int64 + * + * @typedef {object} RecordBatch + * @param {string} producerId Int64 + * @param {boolean} inTransaction + */ +module.exports = ({ messages, abortedTransactions }) => { + const currentAbortedTransactions = new Map() + + if (!abortedTransactions || !abortedTransactions.length) { + return messages + } + + const remainingAbortedTransactions = [...abortedTransactions] + + return messages.filter(message => { + // If the message offset is GTE the first offset of the next aborted transaction + // then we have stepped into an aborted transaction. + if ( + remainingAbortedTransactions.length && + Long.fromValue(message.offset).gte(remainingAbortedTransactions[0].firstOffset) + ) { + const { producerId } = remainingAbortedTransactions.shift() + currentAbortedTransactions.set(producerId, true) + } + + const { producerId, inTransaction } = message.batchContext + + if (isAbortMarker(message)) { + // Transaction is over, we no longer need to ignore messages from this producer + currentAbortedTransactions.delete(producerId) + } else if (currentAbortedTransactions.has(producerId) && inTransaction) { + return false + } + + return true + }) +} diff --git a/node_modules/kafkajs/src/consumer/index.js b/node_modules/kafkajs/src/consumer/index.js new file mode 100644 index 0000000..0489dcf --- /dev/null +++ b/node_modules/kafkajs/src/consumer/index.js @@ -0,0 +1,522 @@ +const Long = require('../utils/long') +const createRetry = require('../retry') +const { initialRetryTime } = require('../retry/defaults') +const ConsumerGroup = require('./consumerGroup') +const Runner = require('./runner') +const { events, wrap: wrapEvent, unwrap: unwrapEvent } = require('./instrumentationEvents') +const InstrumentationEventEmitter = require('../instrumentation/emitter') +const { KafkaJSNonRetriableError } = require('../errors') +const { roundRobin } = require('./assigners') +const { EARLIEST_OFFSET, LATEST_OFFSET } = require('../constants') +const ISOLATION_LEVEL = require('../protocol/isolationLevel') +const sharedPromiseTo = require('../utils/sharedPromiseTo') + +const { keys, values } = Object +const { CONNECT, DISCONNECT, STOP, CRASH } = events + +const eventNames = values(events) +const eventKeys = keys(events) + .map(key => `consumer.events.${key}`) + .join(', ') + +const specialOffsets = [ + Long.fromValue(EARLIEST_OFFSET).toString(), + Long.fromValue(LATEST_OFFSET).toString(), +] + +/** + * @param {Object} params + * @param {import("../../types").Cluster} params.cluster + * @param {String} params.groupId + * @param {import('../../types').RetryOptions} [params.retry] + * @param {import('../../types').Logger} params.logger + * @param {import('../../types').PartitionAssigner[]} [params.partitionAssigners] + * @param {number} [params.sessionTimeout] + * @param {number} [params.rebalanceTimeout] + * @param {number} [params.heartbeatInterval] + * @param {number} [params.maxBytesPerPartition] + * @param {number} [params.minBytes] + * @param {number} [params.maxBytes] + * @param {number} [params.maxWaitTimeInMs] + * @param {number} [params.isolationLevel] + * @param {string} [params.rackId] + * @param {InstrumentationEventEmitter} [params.instrumentationEmitter] + * @param {number} params.metadataMaxAge + * + * @returns {import("../../types").Consumer} + */ +module.exports = ({ + cluster, + groupId, + retry, + logger: rootLogger, + partitionAssigners = [roundRobin], + sessionTimeout = 30000, + rebalanceTimeout = 60000, + heartbeatInterval = 3000, + maxBytesPerPartition = 1048576, // 1MB + minBytes = 1, + maxBytes = 10485760, // 10MB + maxWaitTimeInMs = 5000, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + rackId = '', + instrumentationEmitter: rootInstrumentationEmitter, + metadataMaxAge, +}) => { + if (!groupId) { + throw new KafkaJSNonRetriableError('Consumer groupId must be a non-empty string.') + } + + const logger = rootLogger.namespace('Consumer') + const instrumentationEmitter = rootInstrumentationEmitter || new InstrumentationEventEmitter() + const assigners = partitionAssigners.map(createAssigner => + createAssigner({ groupId, logger, cluster }) + ) + + /** @type {Record} */ + const topics = {} + let runner = null + /** @type {ConsumerGroup} */ + let consumerGroup = null + let restartTimeout = null + + if (heartbeatInterval >= sessionTimeout) { + throw new KafkaJSNonRetriableError( + `Consumer heartbeatInterval (${heartbeatInterval}) must be lower than sessionTimeout (${sessionTimeout}). It is recommended to set heartbeatInterval to approximately a third of the sessionTimeout.` + ) + } + + /** @type {import("../../types").Consumer["connect"]} */ + const connect = async () => { + await cluster.connect() + instrumentationEmitter.emit(CONNECT) + } + + /** @type {import("../../types").Consumer["disconnect"]} */ + const disconnect = async () => { + try { + await stop() + logger.debug('consumer has stopped, disconnecting', { groupId }) + await cluster.disconnect() + instrumentationEmitter.emit(DISCONNECT) + } catch (e) { + logger.error(`Caught error when disconnecting the consumer: ${e.message}`, { + stack: e.stack, + groupId, + }) + throw e + } + } + + /** @type {import("../../types").Consumer["stop"]} */ + const stop = sharedPromiseTo(async () => { + try { + if (runner) { + await runner.stop() + runner = null + consumerGroup = null + instrumentationEmitter.emit(STOP) + } + + clearTimeout(restartTimeout) + logger.info('Stopped', { groupId }) + } catch (e) { + logger.error(`Caught error when stopping the consumer: ${e.message}`, { + stack: e.stack, + groupId, + }) + + throw e + } + }) + + /** @type {import("../../types").Consumer["subscribe"]} */ + const subscribe = async ({ topic, topics: subscriptionTopics, fromBeginning = false }) => { + if (consumerGroup) { + throw new KafkaJSNonRetriableError('Cannot subscribe to topic while consumer is running') + } + + if (!topic && !subscriptionTopics) { + throw new KafkaJSNonRetriableError('Missing required argument "topics"') + } + + if (subscriptionTopics != null && !Array.isArray(subscriptionTopics)) { + throw new KafkaJSNonRetriableError('Argument "topics" must be an array') + } + + const subscriptions = subscriptionTopics || [topic] + + for (const subscription of subscriptions) { + if (typeof subscription !== 'string' && !(subscription instanceof RegExp)) { + throw new KafkaJSNonRetriableError( + `Invalid topic ${subscription} (${typeof subscription}), the topic name has to be a String or a RegExp` + ) + } + } + + const hasRegexSubscriptions = subscriptions.some(subscription => subscription instanceof RegExp) + const metadata = hasRegexSubscriptions ? await cluster.metadata() : undefined + + const topicsToSubscribe = [] + for (const subscription of subscriptions) { + const isRegExp = subscription instanceof RegExp + if (isRegExp) { + const topicRegExp = subscription + const matchedTopics = metadata.topicMetadata + .map(({ topic: topicName }) => topicName) + .filter(topicName => topicRegExp.test(topicName)) + + logger.debug('Subscription based on RegExp', { + groupId, + topicRegExp: topicRegExp.toString(), + matchedTopics, + }) + + topicsToSubscribe.push(...matchedTopics) + } else { + topicsToSubscribe.push(subscription) + } + } + + for (const t of topicsToSubscribe) { + topics[t] = { fromBeginning } + } + + await cluster.addMultipleTargetTopics(topicsToSubscribe) + } + + /** @type {import("../../types").Consumer["run"]} */ + const run = async ({ + autoCommit = true, + autoCommitInterval = null, + autoCommitThreshold = null, + eachBatchAutoResolve = true, + partitionsConsumedConcurrently: concurrency = 1, + eachBatch = null, + eachMessage = null, + } = {}) => { + if (consumerGroup) { + logger.warn('consumer#run was called, but the consumer is already running', { groupId }) + return + } + + const start = async onCrash => { + logger.info('Starting', { groupId }) + + consumerGroup = new ConsumerGroup({ + logger: rootLogger, + topics: keys(topics), + topicConfigurations: topics, + retry, + cluster, + groupId, + assigners, + sessionTimeout, + rebalanceTimeout, + maxBytesPerPartition, + minBytes, + maxBytes, + maxWaitTimeInMs, + instrumentationEmitter, + isolationLevel, + rackId, + metadataMaxAge, + autoCommit, + autoCommitInterval, + autoCommitThreshold, + }) + + runner = new Runner({ + logger: rootLogger, + consumerGroup, + instrumentationEmitter, + heartbeatInterval, + retry, + autoCommit, + eachBatchAutoResolve, + eachBatch, + eachMessage, + onCrash, + concurrency, + }) + + await runner.start() + } + + const onCrash = async e => { + logger.error(`Crash: ${e.name}: ${e.message}`, { + groupId, + retryCount: e.retryCount, + stack: e.stack, + }) + + if (e.name === 'KafkaJSConnectionClosedError') { + cluster.removeBroker({ host: e.host, port: e.port }) + } + + await disconnect() + + const getOriginalCause = error => { + if (error.cause) { + return getOriginalCause(error.cause) + } + + return error + } + + const isErrorRetriable = + e.name === 'KafkaJSNumberOfRetriesExceeded' || getOriginalCause(e).retriable === true + const shouldRestart = + isErrorRetriable && + (!retry || + !retry.restartOnFailure || + (await retry.restartOnFailure(e).catch(error => { + logger.error( + 'Caught error when invoking user-provided "restartOnFailure" callback. Defaulting to restarting.', + { + error: error.message || error, + cause: e.message || e, + groupId, + } + ) + + return true + }))) + + instrumentationEmitter.emit(CRASH, { + error: e, + groupId, + restart: shouldRestart, + }) + + if (shouldRestart) { + const retryTime = e.retryTime || (retry && retry.initialRetryTime) || initialRetryTime + logger.error(`Restarting the consumer in ${retryTime}ms`, { + retryCount: e.retryCount, + retryTime, + groupId, + }) + + restartTimeout = setTimeout(() => start(onCrash), retryTime) + } + } + + await start(onCrash) + } + + /** @type {import("../../types").Consumer["on"]} */ + const on = (eventName, listener) => { + if (!eventNames.includes(eventName)) { + throw new KafkaJSNonRetriableError(`Event name should be one of ${eventKeys}`) + } + + return instrumentationEmitter.addListener(unwrapEvent(eventName), event => { + event.type = wrapEvent(event.type) + Promise.resolve(listener(event)).catch(e => { + logger.error(`Failed to execute listener: ${e.message}`, { + eventName, + stack: e.stack, + }) + }) + }) + } + + /** + * @type {import("../../types").Consumer["commitOffsets"]} + * @param topicPartitions + * Example: [{ topic: 'topic-name', partition: 0, offset: '1', metadata: 'event-id-3' }] + */ + const commitOffsets = async (topicPartitions = []) => { + const commitsByTopic = topicPartitions.reduce( + (payload, { topic, partition, offset, metadata = null }) => { + if (!topic) { + throw new KafkaJSNonRetriableError(`Invalid topic ${topic}`) + } + + if (isNaN(partition)) { + throw new KafkaJSNonRetriableError( + `Invalid partition, expected a number received ${partition}` + ) + } + + let commitOffset + try { + commitOffset = Long.fromValue(offset) + } catch (_) { + throw new KafkaJSNonRetriableError(`Invalid offset, expected a long received ${offset}`) + } + + if (commitOffset.lessThan(0)) { + throw new KafkaJSNonRetriableError('Offset must not be a negative number') + } + + if (metadata !== null && typeof metadata !== 'string') { + throw new KafkaJSNonRetriableError( + `Invalid offset metadata, expected string or null, received ${metadata}` + ) + } + + const topicCommits = payload[topic] || [] + + topicCommits.push({ partition, offset: commitOffset, metadata }) + + return { ...payload, [topic]: topicCommits } + }, + {} + ) + + if (!consumerGroup) { + throw new KafkaJSNonRetriableError( + 'Consumer group was not initialized, consumer#run must be called first' + ) + } + + const topics = Object.keys(commitsByTopic) + + return runner.commitOffsets({ + topics: topics.map(topic => { + return { + topic, + partitions: commitsByTopic[topic], + } + }), + }) + } + + /** @type {import("../../types").Consumer["seek"]} */ + const seek = ({ topic, partition, offset }) => { + if (!topic) { + throw new KafkaJSNonRetriableError(`Invalid topic ${topic}`) + } + + if (isNaN(partition)) { + throw new KafkaJSNonRetriableError( + `Invalid partition, expected a number received ${partition}` + ) + } + + let seekOffset + try { + seekOffset = Long.fromValue(offset) + } catch (_) { + throw new KafkaJSNonRetriableError(`Invalid offset, expected a long received ${offset}`) + } + + if (seekOffset.lessThan(0) && !specialOffsets.includes(seekOffset.toString())) { + throw new KafkaJSNonRetriableError('Offset must not be a negative number') + } + + if (!consumerGroup) { + throw new KafkaJSNonRetriableError( + 'Consumer group was not initialized, consumer#run must be called first' + ) + } + + consumerGroup.seek({ topic, partition, offset: seekOffset.toString() }) + } + + /** @type {import("../../types").Consumer["describeGroup"]} */ + const describeGroup = async () => { + const coordinator = await cluster.findGroupCoordinator({ groupId }) + const retrier = createRetry(retry) + return retrier(async () => { + const { groups } = await coordinator.describeGroups({ groupIds: [groupId] }) + return groups.find(group => group.groupId === groupId) + }) + } + + /** + * @type {import("../../types").Consumer["pause"]} + * @param topicPartitions + * Example: [{ topic: 'topic-name', partitions: [1, 2] }] + */ + const pause = (topicPartitions = []) => { + for (const topicPartition of topicPartitions) { + if (!topicPartition || !topicPartition.topic) { + throw new KafkaJSNonRetriableError( + `Invalid topic ${(topicPartition && topicPartition.topic) || topicPartition}` + ) + } else if ( + typeof topicPartition.partitions !== 'undefined' && + (!Array.isArray(topicPartition.partitions) || topicPartition.partitions.some(isNaN)) + ) { + throw new KafkaJSNonRetriableError( + `Array of valid partitions required to pause specific partitions instead of ${topicPartition.partitions}` + ) + } + } + + if (!consumerGroup) { + throw new KafkaJSNonRetriableError( + 'Consumer group was not initialized, consumer#run must be called first' + ) + } + + consumerGroup.pause(topicPartitions) + } + + /** + * Returns the list of topic partitions paused on this consumer + * + * @type {import("../../types").Consumer["paused"]} + */ + const paused = () => { + if (!consumerGroup) { + return [] + } + + return consumerGroup.paused() + } + + /** + * @type {import("../../types").Consumer["resume"]} + * @param topicPartitions + * Example: [{ topic: 'topic-name', partitions: [1, 2] }] + */ + const resume = (topicPartitions = []) => { + for (const topicPartition of topicPartitions) { + if (!topicPartition || !topicPartition.topic) { + throw new KafkaJSNonRetriableError( + `Invalid topic ${(topicPartition && topicPartition.topic) || topicPartition}` + ) + } else if ( + typeof topicPartition.partitions !== 'undefined' && + (!Array.isArray(topicPartition.partitions) || topicPartition.partitions.some(isNaN)) + ) { + throw new KafkaJSNonRetriableError( + `Array of valid partitions required to resume specific partitions instead of ${topicPartition.partitions}` + ) + } + } + + if (!consumerGroup) { + throw new KafkaJSNonRetriableError( + 'Consumer group was not initialized, consumer#run must be called first' + ) + } + + consumerGroup.resume(topicPartitions) + } + + /** + * @return {Object} logger + */ + const getLogger = () => logger + + return { + connect, + disconnect, + subscribe, + stop, + run, + commitOffsets, + seek, + describeGroup, + pause, + paused, + resume, + on, + events, + logger: getLogger, + } +} diff --git a/node_modules/kafkajs/src/consumer/instrumentationEvents.js b/node_modules/kafkajs/src/consumer/instrumentationEvents.js new file mode 100644 index 0000000..6d7f616 --- /dev/null +++ b/node_modules/kafkajs/src/consumer/instrumentationEvents.js @@ -0,0 +1,40 @@ +const swapObject = require('../utils/swapObject') +const InstrumentationEventType = require('../instrumentation/eventType') +const networkEvents = require('../network/instrumentationEvents') +const consumerType = InstrumentationEventType('consumer') + +/** @type {import('types').ConsumerEvents} */ +const events = { + HEARTBEAT: consumerType('heartbeat'), + COMMIT_OFFSETS: consumerType('commit_offsets'), + GROUP_JOIN: consumerType('group_join'), + FETCH: consumerType('fetch'), + FETCH_START: consumerType('fetch_start'), + START_BATCH_PROCESS: consumerType('start_batch_process'), + END_BATCH_PROCESS: consumerType('end_batch_process'), + CONNECT: consumerType('connect'), + DISCONNECT: consumerType('disconnect'), + STOP: consumerType('stop'), + CRASH: consumerType('crash'), + REBALANCING: consumerType('rebalancing'), + RECEIVED_UNSUBSCRIBED_TOPICS: consumerType('received_unsubscribed_topics'), + REQUEST: consumerType(networkEvents.NETWORK_REQUEST), + REQUEST_TIMEOUT: consumerType(networkEvents.NETWORK_REQUEST_TIMEOUT), + REQUEST_QUEUE_SIZE: consumerType(networkEvents.NETWORK_REQUEST_QUEUE_SIZE), +} + +const wrappedEvents = { + [events.REQUEST]: networkEvents.NETWORK_REQUEST, + [events.REQUEST_TIMEOUT]: networkEvents.NETWORK_REQUEST_TIMEOUT, + [events.REQUEST_QUEUE_SIZE]: networkEvents.NETWORK_REQUEST_QUEUE_SIZE, +} + +const reversedWrappedEvents = swapObject(wrappedEvents) +const unwrap = eventName => wrappedEvents[eventName] || eventName +const wrap = eventName => reversedWrappedEvents[eventName] || eventName + +module.exports = { + events, + wrap, + unwrap, +} diff --git a/node_modules/kafkajs/src/consumer/offsetManager/index.js b/node_modules/kafkajs/src/consumer/offsetManager/index.js new file mode 100644 index 0000000..9077a9b --- /dev/null +++ b/node_modules/kafkajs/src/consumer/offsetManager/index.js @@ -0,0 +1,384 @@ +const Long = require('../../utils/long') +const isInvalidOffset = require('./isInvalidOffset') +const initializeConsumerOffsets = require('./initializeConsumerOffsets') +const { + events: { COMMIT_OFFSETS }, +} = require('../instrumentationEvents') + +const { keys, assign } = Object +const indexTopics = topics => topics.reduce((obj, topic) => assign(obj, { [topic]: {} }), {}) + +const PRIVATE = { + COMMITTED_OFFSETS: Symbol('private:OffsetManager:committedOffsets'), +} +module.exports = class OffsetManager { + /** + * @param {Object} options + * @param {import("../../../types").Cluster} options.cluster + * @param {import("../../../types").Broker} options.coordinator + * @param {import("../../../types").IMemberAssignment} options.memberAssignment + * @param {boolean} options.autoCommit + * @param {number | null} options.autoCommitInterval + * @param {number | null} options.autoCommitThreshold + * @param {{[topic: string]: { fromBeginning: boolean }}} options.topicConfigurations + * @param {import("../../instrumentation/emitter")} options.instrumentationEmitter + * @param {string} options.groupId + * @param {number} options.generationId + * @param {string} options.memberId + */ + constructor({ + cluster, + coordinator, + memberAssignment, + autoCommit, + autoCommitInterval, + autoCommitThreshold, + topicConfigurations, + instrumentationEmitter, + groupId, + generationId, + memberId, + }) { + this.cluster = cluster + this.coordinator = coordinator + + // memberAssignment format: + // { + // 'topic1': [0, 1, 2, 3], + // 'topic2': [0, 1, 2, 3, 4, 5], + // } + this.memberAssignment = memberAssignment + + this.topicConfigurations = topicConfigurations + this.instrumentationEmitter = instrumentationEmitter + this.groupId = groupId + this.generationId = generationId + this.memberId = memberId + + this.autoCommit = autoCommit + this.autoCommitInterval = autoCommitInterval + this.autoCommitThreshold = autoCommitThreshold + this.lastCommit = Date.now() + + this.topics = keys(memberAssignment) + this.clearAllOffsets() + } + + /** + * @param {string} topic + * @param {number} partition + * @returns {Long} + */ + nextOffset(topic, partition) { + if (!this.resolvedOffsets[topic][partition]) { + this.resolvedOffsets[topic][partition] = this.committedOffsets()[topic][partition] + } + + let offset = this.resolvedOffsets[topic][partition] + if (isInvalidOffset(offset)) { + offset = '0' + } + + return Long.fromValue(offset) + } + + /** + * @returns {Promise} + */ + async getCoordinator() { + if (!this.coordinator.isConnected()) { + this.coordinator = await this.cluster.findBroker(this.coordinator) + } + + return this.coordinator + } + + /** + * @param {import("../../../types").TopicPartition} topicPartition + */ + resetOffset({ topic, partition }) { + this.resolvedOffsets[topic][partition] = this.committedOffsets()[topic][partition] + } + + /** + * @param {import("../../../types").TopicPartitionOffset} topicPartitionOffset + */ + resolveOffset({ topic, partition, offset }) { + this.resolvedOffsets[topic][partition] = Long.fromValue(offset) + .add(1) + .toString() + } + + /** + * @returns {Long} + */ + countResolvedOffsets() { + const committedOffsets = this.committedOffsets() + + const subtractOffsets = (resolvedOffset, committedOffset) => { + const resolvedOffsetLong = Long.fromValue(resolvedOffset) + return isInvalidOffset(committedOffset) + ? resolvedOffsetLong + : resolvedOffsetLong.subtract(Long.fromValue(committedOffset)) + } + + const subtractPartitionOffsets = (resolvedTopicOffsets, committedTopicOffsets) => + keys(resolvedTopicOffsets).map(partition => + subtractOffsets(resolvedTopicOffsets[partition], committedTopicOffsets[partition]) + ) + + const subtractTopicOffsets = topic => + subtractPartitionOffsets(this.resolvedOffsets[topic], committedOffsets[topic]) + + const offsetsDiff = this.topics.flatMap(subtractTopicOffsets) + return offsetsDiff.reduce((sum, offset) => sum.add(offset), Long.fromValue(0)) + } + + /** + * @param {import("../../../types").TopicPartition} topicPartition + */ + async setDefaultOffset({ topic, partition }) { + const { groupId, generationId, memberId } = this + const defaultOffset = this.cluster.defaultOffset(this.topicConfigurations[topic]) + const coordinator = await this.getCoordinator() + + await coordinator.offsetCommit({ + groupId, + memberId, + groupGenerationId: generationId, + topics: [ + { + topic, + partitions: [{ partition, offset: defaultOffset }], + }, + ], + }) + + this.clearOffsets({ topic, partition }) + } + + /** + * Commit the given offset to the topic/partition. If the consumer isn't assigned to the given + * topic/partition this method will be a NO-OP. + * + * @param {import("../../../types").TopicPartitionOffset} topicPartitionOffset + */ + async seek({ topic, partition, offset }) { + if (!this.memberAssignment[topic] || !this.memberAssignment[topic].includes(partition)) { + return + } + + if (!this.autoCommit) { + this.resolveOffset({ + topic, + partition, + offset: Long.fromValue(offset) + .subtract(1) + .toString(), + }) + return + } + + const { groupId, generationId, memberId } = this + const coordinator = await this.getCoordinator() + + await coordinator.offsetCommit({ + groupId, + memberId, + groupGenerationId: generationId, + topics: [ + { + topic, + partitions: [{ partition, offset }], + }, + ], + }) + + this.clearOffsets({ topic, partition }) + } + + async commitOffsetsIfNecessary() { + const now = Date.now() + + const timeoutReached = + this.autoCommitInterval != null && now >= this.lastCommit + this.autoCommitInterval + + const thresholdReached = + this.autoCommitThreshold != null && + this.countResolvedOffsets().gte(Long.fromValue(this.autoCommitThreshold)) + + if (timeoutReached || thresholdReached) { + return this.commitOffsets() + } + } + + /** + * Return all locally resolved offsets which are not marked as committed, by topic-partition. + * @returns {import('../../../types').OffsetsByTopicPartition} + */ + uncommittedOffsets() { + const offsets = topic => keys(this.resolvedOffsets[topic]) + const emptyPartitions = ({ partitions }) => partitions.length > 0 + const toPartitions = topic => partition => ({ + partition, + offset: this.resolvedOffsets[topic][partition], + }) + const changedOffsets = topic => ({ partition, offset }) => { + return ( + offset !== this.committedOffsets()[topic][partition] && + Long.fromValue(offset).greaterThanOrEqual(0) + ) + } + + // Select and format updated partitions + const topicsWithPartitionsToCommit = this.topics + .map(topic => ({ + topic, + partitions: offsets(topic) + .map(toPartitions(topic)) + .filter(changedOffsets(topic)), + })) + .filter(emptyPartitions) + + return { topics: topicsWithPartitionsToCommit } + } + + async commitOffsets(offsets = {}) { + const { groupId, generationId, memberId } = this + const { topics = this.uncommittedOffsets().topics } = offsets + + if (topics.length === 0) { + this.lastCommit = Date.now() + return + } + + const payload = { + groupId, + memberId, + groupGenerationId: generationId, + topics, + } + + try { + const coordinator = await this.getCoordinator() + await coordinator.offsetCommit(payload) + this.instrumentationEmitter.emit(COMMIT_OFFSETS, payload) + + // Update local reference of committed offsets + topics.forEach(({ topic, partitions }) => { + const updatedOffsets = partitions.reduce( + (obj, { partition, offset }) => assign(obj, { [partition]: offset }), + {} + ) + + this[PRIVATE.COMMITTED_OFFSETS][topic] = assign( + {}, + this.committedOffsets()[topic], + updatedOffsets + ) + }) + + this.lastCommit = Date.now() + } catch (e) { + // metadata is stale, the coordinator has changed due to a restart or + // broker reassignment + if (e.type === 'NOT_COORDINATOR_FOR_GROUP') { + await this.cluster.refreshMetadata() + } + + throw e + } + } + + async resolveOffsets() { + const { groupId } = this + const invalidOffset = topic => partition => { + return isInvalidOffset(this.committedOffsets()[topic][partition]) + } + + const pendingPartitions = this.topics + .map(topic => ({ + topic, + partitions: this.memberAssignment[topic] + .filter(invalidOffset(topic)) + .map(partition => ({ partition })), + })) + .filter(t => t.partitions.length > 0) + + if (pendingPartitions.length === 0) { + return + } + + const coordinator = await this.getCoordinator() + const { responses: consumerOffsets } = await coordinator.offsetFetch({ + groupId, + topics: pendingPartitions, + }) + + const unresolvedPartitions = consumerOffsets.map(({ topic, partitions }) => + assign( + { + topic, + partitions: partitions + .filter(({ offset }) => isInvalidOffset(offset)) + .map(({ partition }) => assign({ partition })), + }, + this.topicConfigurations[topic] + ) + ) + + const indexPartitions = (obj, { partition, offset }) => { + return assign(obj, { [partition]: offset }) + } + + const hasUnresolvedPartitions = () => unresolvedPartitions.some(t => t.partitions.length > 0) + + let offsets = consumerOffsets + if (hasUnresolvedPartitions()) { + const topicOffsets = await this.cluster.fetchTopicsOffset(unresolvedPartitions) + offsets = initializeConsumerOffsets(consumerOffsets, topicOffsets) + } + + offsets.forEach(({ topic, partitions }) => { + this.committedOffsets()[topic] = partitions.reduce(indexPartitions, { + ...this.committedOffsets()[topic], + }) + }) + } + + /** + * @private + * @param {import("../../../types").TopicPartition} topicPartition + */ + clearOffsets({ topic, partition }) { + delete this.committedOffsets()[topic][partition] + delete this.resolvedOffsets[topic][partition] + } + + /** + * @private + */ + clearAllOffsets() { + const committedOffsets = this.committedOffsets() + + for (const topic in committedOffsets) { + delete committedOffsets[topic] + } + + for (const topic of this.topics) { + committedOffsets[topic] = {} + } + + this.resolvedOffsets = indexTopics(this.topics) + } + + committedOffsets() { + if (!this[PRIVATE.COMMITTED_OFFSETS]) { + this[PRIVATE.COMMITTED_OFFSETS] = this.groupId + ? this.cluster.committedOffsets({ groupId: this.groupId }) + : {} + } + + return this[PRIVATE.COMMITTED_OFFSETS] + } +} diff --git a/node_modules/kafkajs/src/consumer/offsetManager/initializeConsumerOffsets.js b/node_modules/kafkajs/src/consumer/offsetManager/initializeConsumerOffsets.js new file mode 100644 index 0000000..656182e --- /dev/null +++ b/node_modules/kafkajs/src/consumer/offsetManager/initializeConsumerOffsets.js @@ -0,0 +1,26 @@ +const isInvalidOffset = require('./isInvalidOffset') +const { keys, assign } = Object + +const indexPartitions = (obj, { partition, offset }) => assign(obj, { [partition]: offset }) +const indexTopics = (obj, { topic, partitions }) => + assign(obj, { [topic]: partitions.reduce(indexPartitions, {}) }) + +module.exports = (consumerOffsets, topicOffsets) => { + const indexedConsumerOffsets = consumerOffsets.reduce(indexTopics, {}) + const indexedTopicOffsets = topicOffsets.reduce(indexTopics, {}) + + return keys(indexedConsumerOffsets).map(topic => { + const partitions = indexedConsumerOffsets[topic] + return { + topic, + partitions: keys(partitions).map(partition => { + const offset = partitions[partition] + const resolvedOffset = isInvalidOffset(offset) + ? indexedTopicOffsets[topic][partition] + : offset + + return { partition: Number(partition), offset: resolvedOffset } + }), + } + }) +} diff --git a/node_modules/kafkajs/src/consumer/offsetManager/isInvalidOffset.js b/node_modules/kafkajs/src/consumer/offsetManager/isInvalidOffset.js new file mode 100644 index 0000000..0fcc504 --- /dev/null +++ b/node_modules/kafkajs/src/consumer/offsetManager/isInvalidOffset.js @@ -0,0 +1,3 @@ +const Long = require('../../utils/long') + +module.exports = offset => (!offset && offset !== 0) || Long.fromValue(offset).isNegative() diff --git a/node_modules/kafkajs/src/consumer/runner.js b/node_modules/kafkajs/src/consumer/runner.js new file mode 100644 index 0000000..8ced6fa --- /dev/null +++ b/node_modules/kafkajs/src/consumer/runner.js @@ -0,0 +1,518 @@ +const { EventEmitter } = require('events') +const Long = require('../utils/long') +const createRetry = require('../retry') +const { isKafkaJSError, isRebalancing } = require('../errors') + +const { + events: { FETCH, FETCH_START, START_BATCH_PROCESS, END_BATCH_PROCESS, REBALANCING }, +} = require('./instrumentationEvents') +const createFetchManager = require('./fetchManager') + +const isSameOffset = (offsetA, offsetB) => Long.fromValue(offsetA).equals(Long.fromValue(offsetB)) +const CONSUMING_START = 'consuming-start' +const CONSUMING_STOP = 'consuming-stop' + +module.exports = class Runner extends EventEmitter { + /** + * @param {object} options + * @param {import("../../types").Logger} options.logger + * @param {import("./consumerGroup")} options.consumerGroup + * @param {import("../instrumentation/emitter")} options.instrumentationEmitter + * @param {boolean} [options.eachBatchAutoResolve=true] + * @param {number} options.concurrency + * @param {(payload: import("../../types").EachBatchPayload) => Promise} [options.eachBatch] + * @param {(payload: import("../../types").EachMessagePayload) => Promise} [options.eachMessage] + * @param {number} [options.heartbeatInterval] + * @param {(reason: Error) => void} options.onCrash + * @param {import("../../types").RetryOptions} [options.retry] + * @param {boolean} [options.autoCommit=true] + */ + constructor({ + logger, + consumerGroup, + instrumentationEmitter, + eachBatchAutoResolve = true, + concurrency, + eachBatch, + eachMessage, + heartbeatInterval, + onCrash, + retry, + autoCommit = true, + }) { + super() + this.logger = logger.namespace('Runner') + this.consumerGroup = consumerGroup + this.instrumentationEmitter = instrumentationEmitter + this.eachBatchAutoResolve = eachBatchAutoResolve + this.eachBatch = eachBatch + this.eachMessage = eachMessage + this.heartbeatInterval = heartbeatInterval + this.retrier = createRetry(Object.assign({}, retry)) + this.onCrash = onCrash + this.autoCommit = autoCommit + this.fetchManager = createFetchManager({ + logger: this.logger, + getNodeIds: () => this.consumerGroup.getNodeIds(), + fetch: nodeId => this.fetch(nodeId), + handler: batch => this.handleBatch(batch), + concurrency, + }) + + this.running = false + this.consuming = false + } + + get consuming() { + return this._consuming + } + + set consuming(value) { + if (this._consuming !== value) { + this._consuming = value + this.emit(value ? CONSUMING_START : CONSUMING_STOP) + } + } + + async start() { + if (this.running) { + return + } + + try { + await this.consumerGroup.connect() + await this.consumerGroup.joinAndSync() + } catch (e) { + return this.onCrash(e) + } + + this.running = true + this.scheduleFetchManager() + } + + scheduleFetchManager() { + if (!this.running) { + this.consuming = false + + this.logger.info('consumer not running, exiting', { + groupId: this.consumerGroup.groupId, + memberId: this.consumerGroup.memberId, + }) + + return + } + + this.consuming = true + + this.retrier(async (bail, retryCount, retryTime) => { + if (!this.running) { + return + } + + try { + await this.fetchManager.start() + } catch (e) { + if (isRebalancing(e)) { + this.logger.warn('The group is rebalancing, re-joining', { + groupId: this.consumerGroup.groupId, + memberId: this.consumerGroup.memberId, + error: e.message, + }) + + this.instrumentationEmitter.emit(REBALANCING, { + groupId: this.consumerGroup.groupId, + memberId: this.consumerGroup.memberId, + }) + + await this.consumerGroup.joinAndSync() + return + } + + if (e.type === 'UNKNOWN_MEMBER_ID') { + this.logger.error('The coordinator is not aware of this member, re-joining the group', { + groupId: this.consumerGroup.groupId, + memberId: this.consumerGroup.memberId, + error: e.message, + }) + + this.consumerGroup.memberId = null + await this.consumerGroup.joinAndSync() + return + } + + if (e.name === 'KafkaJSNotImplemented') { + return bail(e) + } + + if (e.name === 'KafkaJSNoBrokerAvailableError') { + return bail(e) + } + + this.logger.debug('Error while scheduling fetch manager, trying again...', { + groupId: this.consumerGroup.groupId, + memberId: this.consumerGroup.memberId, + error: e.message, + stack: e.stack, + retryCount, + retryTime, + }) + + throw e + } + }) + .then(() => { + this.scheduleFetchManager() + }) + .catch(e => { + this.onCrash(e) + this.consuming = false + this.running = false + }) + } + + async stop() { + if (!this.running) { + return + } + + this.logger.debug('stop consumer group', { + groupId: this.consumerGroup.groupId, + memberId: this.consumerGroup.memberId, + }) + + this.running = false + + try { + await this.fetchManager.stop() + await this.waitForConsumer() + await this.consumerGroup.leave() + } catch (e) {} + } + + waitForConsumer() { + return new Promise(resolve => { + if (!this.consuming) { + return resolve() + } + + this.logger.debug('waiting for consumer to finish...', { + groupId: this.consumerGroup.groupId, + memberId: this.consumerGroup.memberId, + }) + + this.once(CONSUMING_STOP, () => resolve()) + }) + } + + async heartbeat() { + try { + await this.consumerGroup.heartbeat({ interval: this.heartbeatInterval }) + } catch (e) { + if (isRebalancing(e)) { + await this.autoCommitOffsets() + } + throw e + } + } + + async processEachMessage(batch) { + const { topic, partition } = batch + + const pause = () => { + this.consumerGroup.pause([{ topic, partitions: [partition] }]) + return () => this.consumerGroup.resume([{ topic, partitions: [partition] }]) + } + for (const message of batch.messages) { + if (!this.running || this.consumerGroup.hasSeekOffset({ topic, partition })) { + break + } + + try { + await this.eachMessage({ + topic, + partition, + message, + heartbeat: () => this.heartbeat(), + pause, + }) + } catch (e) { + if (!isKafkaJSError(e)) { + this.logger.error(`Error when calling eachMessage`, { + topic, + partition, + offset: message.offset, + stack: e.stack, + error: e, + }) + } + + // In case of errors, commit the previously consumed offsets unless autoCommit is disabled + await this.autoCommitOffsets() + throw e + } + + this.consumerGroup.resolveOffset({ topic, partition, offset: message.offset }) + await this.heartbeat() + await this.autoCommitOffsetsIfNecessary() + + if (this.consumerGroup.isPaused(topic, partition)) { + break + } + } + } + + async processEachBatch(batch) { + const { topic, partition } = batch + const lastFilteredMessage = batch.messages[batch.messages.length - 1] + + const pause = () => { + this.consumerGroup.pause([{ topic, partitions: [partition] }]) + return () => this.consumerGroup.resume([{ topic, partitions: [partition] }]) + } + + try { + await this.eachBatch({ + batch, + resolveOffset: offset => { + /** + * The transactional producer generates a control record after committing the transaction. + * The control record is the last record on the RecordBatch, and it is filtered before it + * reaches the eachBatch callback. When disabling auto-resolve, the user-land code won't + * be able to resolve the control record offset, since it never reaches the callback, + * causing stuck consumers as the consumer will never move the offset marker. + * + * When the last offset of the batch is resolved, we should automatically resolve + * the control record offset as this entry doesn't have any meaning to the user-land code, + * and won't interfere with the stream processing. + * + * @see https://github.com/apache/kafka/blob/9aa660786e46c1efbf5605a6a69136a1dac6edb9/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java#L1499-L1505 + */ + const offsetToResolve = + lastFilteredMessage && isSameOffset(offset, lastFilteredMessage.offset) + ? batch.lastOffset() + : offset + + this.consumerGroup.resolveOffset({ topic, partition, offset: offsetToResolve }) + }, + heartbeat: () => this.heartbeat(), + /** + * Pause consumption for the current topic-partition being processed + */ + pause, + /** + * Commit offsets if provided. Otherwise commit most recent resolved offsets + * if the autoCommit conditions are met. + * + * @param {import('../../types').OffsetsByTopicPartition} [offsets] Optional. + */ + commitOffsetsIfNecessary: async offsets => { + return offsets + ? this.consumerGroup.commitOffsets(offsets) + : this.consumerGroup.commitOffsetsIfNecessary() + }, + uncommittedOffsets: () => this.consumerGroup.uncommittedOffsets(), + isRunning: () => this.running, + isStale: () => this.consumerGroup.hasSeekOffset({ topic, partition }), + }) + } catch (e) { + if (!isKafkaJSError(e)) { + this.logger.error(`Error when calling eachBatch`, { + topic, + partition, + offset: batch.firstOffset(), + stack: e.stack, + error: e, + }) + } + + // eachBatch has a special resolveOffset which can be used + // to keep track of the messages + await this.autoCommitOffsets() + throw e + } + + // resolveOffset for the last offset can be disabled to allow the users of eachBatch to + // stop their consumers without resolving unprocessed offsets (issues/18) + if (this.eachBatchAutoResolve) { + this.consumerGroup.resolveOffset({ topic, partition, offset: batch.lastOffset() }) + } + } + + async fetch(nodeId) { + if (!this.running) { + this.logger.debug('consumer not running, exiting', { + groupId: this.consumerGroup.groupId, + memberId: this.consumerGroup.memberId, + }) + + return [] + } + + const startFetch = Date.now() + + this.instrumentationEmitter.emit(FETCH_START, { nodeId }) + + const batches = await this.consumerGroup.fetch(nodeId) + + this.instrumentationEmitter.emit(FETCH, { + /** + * PR #570 removed support for the number of batches in this instrumentation event; + * The new implementation uses an async generation to deliver the batches, which makes + * this number impossible to get. The number is set to 0 to keep the event backward + * compatible until we bump KafkaJS to version 2, following the end of node 8 LTS. + * + * @since 2019-11-29 + */ + numberOfBatches: 0, + duration: Date.now() - startFetch, + nodeId, + }) + + if (batches.length === 0) { + await this.heartbeat() + } + + return batches + } + + async handleBatch(batch) { + if (!this.running) { + this.logger.debug('consumer not running, exiting', { + groupId: this.consumerGroup.groupId, + memberId: this.consumerGroup.memberId, + }) + + return + } + + /** @param {import('./batch')} batch */ + const onBatch = async batch => { + const startBatchProcess = Date.now() + const payload = { + topic: batch.topic, + partition: batch.partition, + highWatermark: batch.highWatermark, + offsetLag: batch.offsetLag(), + /** + * @since 2019-06-24 (>= 1.8.0) + * + * offsetLag returns the lag based on the latest offset in the batch, to + * keep the event backward compatible we just introduced "offsetLagLow" + * which calculates the lag based on the first offset in the batch + */ + offsetLagLow: batch.offsetLagLow(), + batchSize: batch.messages.length, + firstOffset: batch.firstOffset(), + lastOffset: batch.lastOffset(), + } + + /** + * If the batch contained only control records or only aborted messages then we still + * need to resolve and auto-commit to ensure the consumer can move forward. + * + * We also need to emit batch instrumentation events to allow any listeners keeping + * track of offsets to know about the latest point of consumption. + * + * Added in #1256 + * + * @see https://github.com/apache/kafka/blob/9aa660786e46c1efbf5605a6a69136a1dac6edb9/clients/src/main/java/org/apache/kafka/clients/consumer/internals/Fetcher.java#L1499-L1505 + */ + if (batch.isEmptyDueToFiltering()) { + this.instrumentationEmitter.emit(START_BATCH_PROCESS, payload) + + this.consumerGroup.resolveOffset({ + topic: batch.topic, + partition: batch.partition, + offset: batch.lastOffset(), + }) + await this.autoCommitOffsetsIfNecessary() + + this.instrumentationEmitter.emit(END_BATCH_PROCESS, { + ...payload, + duration: Date.now() - startBatchProcess, + }) + + await this.heartbeat() + return + } + + if (batch.isEmpty()) { + await this.heartbeat() + return + } + + this.instrumentationEmitter.emit(START_BATCH_PROCESS, payload) + + if (this.eachMessage) { + await this.processEachMessage(batch) + } else if (this.eachBatch) { + await this.processEachBatch(batch) + } + + this.instrumentationEmitter.emit(END_BATCH_PROCESS, { + ...payload, + duration: Date.now() - startBatchProcess, + }) + + await this.autoCommitOffsets() + await this.heartbeat() + } + + await onBatch(batch) + } + + autoCommitOffsets() { + if (this.autoCommit) { + return this.consumerGroup.commitOffsets() + } + } + + autoCommitOffsetsIfNecessary() { + if (this.autoCommit) { + return this.consumerGroup.commitOffsetsIfNecessary() + } + } + + commitOffsets(offsets) { + if (!this.running) { + this.logger.debug('consumer not running, exiting', { + groupId: this.consumerGroup.groupId, + memberId: this.consumerGroup.memberId, + offsets, + }) + return + } + + return this.retrier(async (bail, retryCount, retryTime) => { + try { + await this.consumerGroup.commitOffsets(offsets) + } catch (e) { + if (!this.running) { + this.logger.debug('consumer not running, exiting', { + error: e.message, + groupId: this.consumerGroup.groupId, + memberId: this.consumerGroup.memberId, + offsets, + }) + return + } + + if (e.name === 'KafkaJSNotImplemented') { + return bail(e) + } + + this.logger.debug('Error while committing offsets, trying again...', { + groupId: this.consumerGroup.groupId, + memberId: this.consumerGroup.memberId, + error: e.message, + stack: e.stack, + retryCount, + retryTime, + offsets, + }) + + throw e + } + }) + } +} diff --git a/node_modules/kafkajs/src/consumer/seekOffsets.js b/node_modules/kafkajs/src/consumer/seekOffsets.js new file mode 100644 index 0000000..8d0fdd7 --- /dev/null +++ b/node_modules/kafkajs/src/consumer/seekOffsets.js @@ -0,0 +1,27 @@ +module.exports = class SeekOffsets extends Map { + getKey(topic, partition) { + return JSON.stringify([topic, partition]) + } + + set(topic, partition, offset) { + const key = this.getKey(topic, partition) + super.set(key, offset) + } + + has(topic, partition) { + const key = this.getKey(topic, partition) + return super.has(key) + } + + pop(topic, partition) { + if (this.size === 0 || !this.has(topic, partition)) { + return + } + + const key = this.getKey(topic, partition) + const offset = this.get(key) + + this.delete(key) + return { topic, partition, offset } + } +} diff --git a/node_modules/kafkajs/src/consumer/subscriptionState.js b/node_modules/kafkajs/src/consumer/subscriptionState.js new file mode 100644 index 0000000..307d0d0 --- /dev/null +++ b/node_modules/kafkajs/src/consumer/subscriptionState.js @@ -0,0 +1,123 @@ +const createState = topic => ({ + topic, + paused: new Set(), + pauseAll: false, + resumed: new Set(), +}) + +module.exports = class SubscriptionState { + constructor() { + this.assignedPartitionsByTopic = {} + this.subscriptionStatesByTopic = {} + } + + /** + * Replace the current assignment with a new set of assignments + * + * @param {Array} topicPartitions Example: [{ topic: 'topic-name', partitions: [1, 2] }] + */ + assign(topicPartitions = []) { + this.assignedPartitionsByTopic = topicPartitions.reduce( + (assigned, { topic, partitions = [] }) => { + return { ...assigned, [topic]: { topic, partitions } } + }, + {} + ) + } + + /** + * @param {Array} topicPartitions Example: [{ topic: 'topic-name', partitions: [1, 2] }] + */ + pause(topicPartitions = []) { + topicPartitions.forEach(({ topic, partitions }) => { + const state = this.subscriptionStatesByTopic[topic] || createState(topic) + + if (typeof partitions === 'undefined') { + state.paused.clear() + state.resumed.clear() + state.pauseAll = true + } else if (Array.isArray(partitions)) { + partitions.forEach(partition => { + state.paused.add(partition) + state.resumed.delete(partition) + }) + state.pauseAll = false + } + + this.subscriptionStatesByTopic[topic] = state + }) + } + + /** + * @param {Array} topicPartitions Example: [{ topic: 'topic-name', partitions: [1, 2] }] + */ + resume(topicPartitions = []) { + topicPartitions.forEach(({ topic, partitions }) => { + const state = this.subscriptionStatesByTopic[topic] || createState(topic) + + if (typeof partitions === 'undefined') { + state.paused.clear() + state.resumed.clear() + state.pauseAll = false + } else if (Array.isArray(partitions)) { + partitions.forEach(partition => { + state.paused.delete(partition) + + if (state.pauseAll) { + state.resumed.add(partition) + } + }) + } + + this.subscriptionStatesByTopic[topic] = state + }) + } + + /** + * @returns {Array} topicPartitions + * Example: [{ topic: 'topic-name', partitions: [1, 2] }] + */ + assigned() { + return Object.values(this.assignedPartitionsByTopic).map(({ topic, partitions }) => ({ + topic, + partitions: partitions.sort(), + })) + } + + /** + * @returns {Array} topicPartitions + * Example: [{ topic: 'topic-name', partitions: [1, 2] }] + */ + active() { + return Object.values(this.assignedPartitionsByTopic).map(({ topic, partitions }) => ({ + topic, + partitions: partitions.filter(partition => !this.isPaused(topic, partition)).sort(), + })) + } + + /** + * @returns {Array} topicPartitions + * Example: [{ topic: 'topic-name', partitions: [1, 2] }] + */ + paused() { + return Object.values(this.assignedPartitionsByTopic) + .map(({ topic, partitions }) => ({ + topic, + partitions: partitions.filter(partition => this.isPaused(topic, partition)).sort(), + })) + .filter(({ partitions }) => partitions.length !== 0) + } + + isPaused(topic, partition) { + const state = this.subscriptionStatesByTopic[topic] + + if (!state) { + return false + } + + const partitionResumed = state.resumed.has(partition) + const partitionPaused = state.paused.has(partition) + + return (state.pauseAll && !partitionResumed) || partitionPaused + } +} diff --git a/node_modules/kafkajs/src/consumer/worker.js b/node_modules/kafkajs/src/consumer/worker.js new file mode 100644 index 0000000..92338ee --- /dev/null +++ b/node_modules/kafkajs/src/consumer/worker.js @@ -0,0 +1,40 @@ +/** + * @typedef {(batch: T, metadata: { workerId: number }) => Promise} Handler + * @template T + * + * @typedef {ReturnType} Worker + */ + +const sharedPromiseTo = require('../utils/sharedPromiseTo') + +/** + * @param {{ handler: Handler, workerId: number }} options + * @template T + */ +const createWorker = ({ handler, workerId }) => { + /** + * Takes batches from next() until it returns undefined. + * + * @param {{ next: () => { batch: T, resolve: () => void, reject: (e: Error) => void } | undefined }} param0 + * @returns {Promise} + */ + const run = sharedPromiseTo(async ({ next }) => { + while (true) { + const item = next() + if (!item) break + + const { batch, resolve, reject } = item + + try { + await handler(batch, { workerId }) + resolve() + } catch (error) { + reject(error) + } + } + }) + + return { run } +} + +module.exports = createWorker diff --git a/node_modules/kafkajs/src/consumer/workerQueue.js b/node_modules/kafkajs/src/consumer/workerQueue.js new file mode 100644 index 0000000..afac172 --- /dev/null +++ b/node_modules/kafkajs/src/consumer/workerQueue.js @@ -0,0 +1,40 @@ +/** + * @typedef {ReturnType} WorkerQueue + */ + +/** + * @param {object} options + * @param {import('./worker').Worker[]} options.workers + * @template T + */ +const createWorkerQueue = ({ workers }) => { + /** @type {{ batch: T, resolve: (value?: any) => void, reject: (e: Error) => void}[]} */ + const queue = [] + + const getWorkers = () => workers + + /** + * Waits until workers have processed all batches in the queue. + * + * @param {...T} batches + * @returns {Promise} + */ + const push = async (...batches) => { + const promises = batches.map( + batch => new Promise((resolve, reject) => queue.push({ batch, resolve, reject })) + ) + + workers.forEach(worker => worker.run({ next: () => queue.shift() })) + + const results = await Promise.allSettled(promises) + const rejected = results.find(result => result.status === 'rejected') + if (rejected) { + // @ts-ignore + throw rejected.reason + } + } + + return { push, getWorkers } +} + +module.exports = createWorkerQueue diff --git a/node_modules/kafkajs/src/env.js b/node_modules/kafkajs/src/env.js new file mode 100644 index 0000000..98ad02a --- /dev/null +++ b/node_modules/kafkajs/src/env.js @@ -0,0 +1,4 @@ +module.exports = () => ({ + KAFKAJS_DEBUG_PROTOCOL_BUFFERS: process.env.KAFKAJS_DEBUG_PROTOCOL_BUFFERS, + KAFKAJS_DEBUG_EXTENDED_PROTOCOL_BUFFERS: process.env.KAFKAJS_DEBUG_EXTENDED_PROTOCOL_BUFFERS, +}) diff --git a/node_modules/kafkajs/src/errors.js b/node_modules/kafkajs/src/errors.js new file mode 100644 index 0000000..266192e --- /dev/null +++ b/node_modules/kafkajs/src/errors.js @@ -0,0 +1,309 @@ +const pkgJson = require('../package.json') +const { bugs } = pkgJson + +class KafkaJSError extends Error { + constructor(e, { retriable = true, cause } = {}) { + super(e, { cause }) + Error.captureStackTrace(this, this.constructor) + this.message = e.message || e + this.name = 'KafkaJSError' + this.retriable = retriable + this.helpUrl = e.helpUrl + this.cause = cause + } +} + +class KafkaJSNonRetriableError extends KafkaJSError { + constructor(e, { cause } = {}) { + super(e, { retriable: false, cause }) + this.name = 'KafkaJSNonRetriableError' + } +} + +class KafkaJSProtocolError extends KafkaJSError { + constructor(e, { retriable = e.retriable } = {}) { + super(e, { retriable }) + this.type = e.type + this.code = e.code + this.name = 'KafkaJSProtocolError' + } +} + +class KafkaJSOffsetOutOfRange extends KafkaJSProtocolError { + constructor(e, { topic, partition }) { + super(e) + this.topic = topic + this.partition = partition + this.name = 'KafkaJSOffsetOutOfRange' + } +} + +class KafkaJSMemberIdRequired extends KafkaJSProtocolError { + constructor(e, { memberId }) { + super(e) + this.memberId = memberId + this.name = 'KafkaJSMemberIdRequired' + } +} + +class KafkaJSNumberOfRetriesExceeded extends KafkaJSNonRetriableError { + constructor(e, { retryCount, retryTime }) { + super(e, { cause: e }) + this.stack = `${this.name}\n Caused by: ${e.stack}` + this.retryCount = retryCount + this.retryTime = retryTime + this.name = 'KafkaJSNumberOfRetriesExceeded' + } +} + +class KafkaJSConnectionError extends KafkaJSError { + /** + * @param {string} e + * @param {object} options + * @param {string} [options.broker] + * @param {string} [options.code] + */ + constructor(e, { broker, code } = {}) { + super(e) + this.broker = broker + this.code = code + this.name = 'KafkaJSConnectionError' + } +} + +class KafkaJSConnectionClosedError extends KafkaJSConnectionError { + constructor(e, { host, port } = {}) { + super(e, { broker: `${host}:${port}` }) + this.host = host + this.port = port + this.name = 'KafkaJSConnectionClosedError' + } +} + +class KafkaJSRequestTimeoutError extends KafkaJSError { + constructor(e, { broker, correlationId, createdAt, sentAt, pendingDuration } = {}) { + super(e) + this.broker = broker + this.correlationId = correlationId + this.createdAt = createdAt + this.sentAt = sentAt + this.pendingDuration = pendingDuration + this.name = 'KafkaJSRequestTimeoutError' + } +} + +class KafkaJSMetadataNotLoaded extends KafkaJSError { + constructor() { + super(...arguments) + this.name = 'KafkaJSMetadataNotLoaded' + } +} +class KafkaJSTopicMetadataNotLoaded extends KafkaJSMetadataNotLoaded { + constructor(e, { topic } = {}) { + super(e) + this.topic = topic + this.name = 'KafkaJSTopicMetadataNotLoaded' + } +} +class KafkaJSStaleTopicMetadataAssignment extends KafkaJSError { + constructor(e, { topic, unknownPartitions } = {}) { + super(e) + this.topic = topic + this.unknownPartitions = unknownPartitions + this.name = 'KafkaJSStaleTopicMetadataAssignment' + } +} + +class KafkaJSDeleteGroupsError extends KafkaJSError { + constructor(e, groups = []) { + super(e) + this.groups = groups + this.name = 'KafkaJSDeleteGroupsError' + } +} + +class KafkaJSServerDoesNotSupportApiKey extends KafkaJSNonRetriableError { + constructor(e, { apiKey, apiName } = {}) { + super(e) + this.apiKey = apiKey + this.apiName = apiName + this.name = 'KafkaJSServerDoesNotSupportApiKey' + } +} + +class KafkaJSBrokerNotFound extends KafkaJSError { + constructor() { + super(...arguments) + this.name = 'KafkaJSBrokerNotFound' + } +} + +class KafkaJSPartialMessageError extends KafkaJSNonRetriableError { + constructor() { + super(...arguments) + this.name = 'KafkaJSPartialMessageError' + } +} + +class KafkaJSSASLAuthenticationError extends KafkaJSNonRetriableError { + constructor() { + super(...arguments) + this.name = 'KafkaJSSASLAuthenticationError' + } +} + +class KafkaJSGroupCoordinatorNotFound extends KafkaJSNonRetriableError { + constructor() { + super(...arguments) + this.name = 'KafkaJSGroupCoordinatorNotFound' + } +} + +class KafkaJSNotImplemented extends KafkaJSNonRetriableError { + constructor() { + super(...arguments) + this.name = 'KafkaJSNotImplemented' + } +} + +class KafkaJSTimeout extends KafkaJSNonRetriableError { + constructor() { + super(...arguments) + this.name = 'KafkaJSTimeout' + } +} + +class KafkaJSLockTimeout extends KafkaJSTimeout { + constructor() { + super(...arguments) + this.name = 'KafkaJSLockTimeout' + } +} + +class KafkaJSUnsupportedMagicByteInMessageSet extends KafkaJSNonRetriableError { + constructor() { + super(...arguments) + this.name = 'KafkaJSUnsupportedMagicByteInMessageSet' + } +} + +class KafkaJSDeleteTopicRecordsError extends KafkaJSError { + constructor({ partitions }) { + /* + * This error is retriable if all the errors were retriable + */ + const retriable = partitions + .filter(({ error }) => error != null) + .every(({ error }) => error.retriable === true) + + super('Error while deleting records', { retriable }) + this.name = 'KafkaJSDeleteTopicRecordsError' + this.partitions = partitions + } +} + +const issueUrl = bugs ? bugs.url : null + +class KafkaJSInvariantViolation extends KafkaJSNonRetriableError { + constructor(e) { + const message = e.message || e + super(`Invariant violated: ${message}. This is likely a bug and should be reported.`) + this.name = 'KafkaJSInvariantViolation' + + if (issueUrl !== null) { + const issueTitle = encodeURIComponent(`Invariant violation: ${message}`) + this.helpUrl = `${issueUrl}/new?assignees=&labels=bug&template=bug_report.md&title=${issueTitle}` + } + } +} + +class KafkaJSInvalidVarIntError extends KafkaJSNonRetriableError { + constructor() { + super(...arguments) + this.name = 'KafkaJSNonRetriableError' + } +} + +class KafkaJSInvalidLongError extends KafkaJSNonRetriableError { + constructor() { + super(...arguments) + this.name = 'KafkaJSNonRetriableError' + } +} + +class KafkaJSCreateTopicError extends KafkaJSProtocolError { + constructor(e, topicName) { + super(e) + this.topic = topicName + this.name = 'KafkaJSCreateTopicError' + } +} + +class KafkaJSAlterPartitionReassignmentsError extends KafkaJSProtocolError { + constructor(e, topicName, partition) { + super(e) + this.topic = topicName + this.partition = partition + this.name = 'KafkaJSAlterPartitionReassignmentsError' + } +} + +class KafkaJSAggregateError extends Error { + constructor(message, errors) { + super(message) + this.errors = errors + this.name = 'KafkaJSAggregateError' + } +} + +class KafkaJSFetcherRebalanceError extends Error {} + +class KafkaJSNoBrokerAvailableError extends KafkaJSError { + constructor() { + super('No broker available') + this.name = 'KafkaJSNoBrokerAvailableError' + } +} + +const isRebalancing = e => + e.type === 'REBALANCE_IN_PROGRESS' || + e.type === 'NOT_COORDINATOR_FOR_GROUP' || + e.type === 'ILLEGAL_GENERATION' + +const isKafkaJSError = e => e instanceof KafkaJSError + +module.exports = { + KafkaJSError, + KafkaJSNonRetriableError, + KafkaJSPartialMessageError, + KafkaJSBrokerNotFound, + KafkaJSProtocolError, + KafkaJSConnectionError, + KafkaJSConnectionClosedError, + KafkaJSRequestTimeoutError, + KafkaJSSASLAuthenticationError, + KafkaJSNumberOfRetriesExceeded, + KafkaJSOffsetOutOfRange, + KafkaJSMemberIdRequired, + KafkaJSGroupCoordinatorNotFound, + KafkaJSNotImplemented, + KafkaJSMetadataNotLoaded, + KafkaJSTopicMetadataNotLoaded, + KafkaJSStaleTopicMetadataAssignment, + KafkaJSDeleteGroupsError, + KafkaJSTimeout, + KafkaJSLockTimeout, + KafkaJSServerDoesNotSupportApiKey, + KafkaJSUnsupportedMagicByteInMessageSet, + KafkaJSDeleteTopicRecordsError, + KafkaJSInvariantViolation, + KafkaJSInvalidVarIntError, + KafkaJSInvalidLongError, + KafkaJSCreateTopicError, + KafkaJSAggregateError, + KafkaJSFetcherRebalanceError, + KafkaJSNoBrokerAvailableError, + KafkaJSAlterPartitionReassignmentsError, + isRebalancing, + isKafkaJSError, +} diff --git a/node_modules/kafkajs/src/index.js b/node_modules/kafkajs/src/index.js new file mode 100644 index 0000000..d6369d7 --- /dev/null +++ b/node_modules/kafkajs/src/index.js @@ -0,0 +1,212 @@ +const { + createLogger, + LEVELS: { INFO }, +} = require('./loggers') + +const InstrumentationEventEmitter = require('./instrumentation/emitter') +const LoggerConsole = require('./loggers/console') +const Cluster = require('./cluster') +const createProducer = require('./producer') +const createConsumer = require('./consumer') +const createAdmin = require('./admin') +const ISOLATION_LEVEL = require('./protocol/isolationLevel') +const defaultSocketFactory = require('./network/socketFactory') +const once = require('./utils/once') +const websiteUrl = require('./utils/websiteUrl') + +const PRIVATE = { + CREATE_CLUSTER: Symbol('private:Kafka:createCluster'), + CLUSTER_RETRY: Symbol('private:Kafka:clusterRetry'), + LOGGER: Symbol('private:Kafka:logger'), + OFFSETS: Symbol('private:Kafka:offsets'), +} + +const DEFAULT_METADATA_MAX_AGE = 300000 +const warnOfDefaultPartitioner = once(logger => { + if (process.env.KAFKAJS_NO_PARTITIONER_WARNING == null) { + logger.warn( + `KafkaJS v2.0.0 switched default partitioner. To retain the same partitioning behavior as in previous versions, create the producer with the option "createPartitioner: Partitioners.LegacyPartitioner". See the migration guide at ${websiteUrl( + 'docs/migration-guide-v2.0.0', + 'producer-new-default-partitioner' + )} for details. Silence this warning by setting the environment variable "KAFKAJS_NO_PARTITIONER_WARNING=1"` + ) + } +}) + +module.exports = class Client { + /** + * @param {Object} options + * @param {Array} options.brokers example: ['127.0.0.1:9092', '127.0.0.1:9094'] + * @param {Object} options.ssl + * @param {Object} options.sasl + * @param {string} options.clientId + * @param {number} [options.connectionTimeout=1000] - in milliseconds + * @param {number} options.authenticationTimeout - in milliseconds + * @param {number} options.reauthenticationThreshold - in milliseconds + * @param {number} [options.requestTimeout=30000] - in milliseconds + * @param {boolean} [options.enforceRequestTimeout] + * @param {import("../types").RetryOptions} [options.retry] + * @param {import("../types").ISocketFactory} [options.socketFactory] + */ + constructor({ + brokers, + ssl, + sasl, + clientId, + connectionTimeout = 1000, + authenticationTimeout, + reauthenticationThreshold, + requestTimeout, + enforceRequestTimeout = true, + retry, + socketFactory = defaultSocketFactory(), + logLevel = INFO, + logCreator = LoggerConsole, + }) { + this[PRIVATE.OFFSETS] = new Map() + this[PRIVATE.LOGGER] = createLogger({ level: logLevel, logCreator }) + this[PRIVATE.CLUSTER_RETRY] = retry + this[PRIVATE.CREATE_CLUSTER] = ({ + metadataMaxAge, + allowAutoTopicCreation = true, + maxInFlightRequests = null, + instrumentationEmitter = null, + isolationLevel, + }) => + new Cluster({ + logger: this[PRIVATE.LOGGER], + retry: this[PRIVATE.CLUSTER_RETRY], + offsets: this[PRIVATE.OFFSETS], + socketFactory, + brokers, + ssl, + sasl, + clientId, + connectionTimeout, + authenticationTimeout, + reauthenticationThreshold, + requestTimeout, + enforceRequestTimeout, + metadataMaxAge, + instrumentationEmitter, + allowAutoTopicCreation, + maxInFlightRequests, + isolationLevel, + }) + } + + /** + * @public + */ + producer({ + createPartitioner, + retry, + metadataMaxAge = DEFAULT_METADATA_MAX_AGE, + allowAutoTopicCreation, + idempotent, + transactionalId, + transactionTimeout, + maxInFlightRequests, + } = {}) { + const instrumentationEmitter = new InstrumentationEventEmitter() + const cluster = this[PRIVATE.CREATE_CLUSTER]({ + metadataMaxAge, + allowAutoTopicCreation, + maxInFlightRequests, + instrumentationEmitter, + }) + + if (createPartitioner == null) { + warnOfDefaultPartitioner(this[PRIVATE.LOGGER]) + } + + return createProducer({ + retry: { ...this[PRIVATE.CLUSTER_RETRY], ...retry }, + logger: this[PRIVATE.LOGGER], + cluster, + createPartitioner, + idempotent, + transactionalId, + transactionTimeout, + instrumentationEmitter, + }) + } + + /** + * @public + */ + consumer({ + groupId, + partitionAssigners, + metadataMaxAge = DEFAULT_METADATA_MAX_AGE, + sessionTimeout, + rebalanceTimeout, + heartbeatInterval, + maxBytesPerPartition, + minBytes, + maxBytes, + maxWaitTimeInMs, + retry = { retries: 5 }, + allowAutoTopicCreation, + maxInFlightRequests, + readUncommitted = false, + rackId = '', + } = {}) { + const isolationLevel = readUncommitted + ? ISOLATION_LEVEL.READ_UNCOMMITTED + : ISOLATION_LEVEL.READ_COMMITTED + + const instrumentationEmitter = new InstrumentationEventEmitter() + const cluster = this[PRIVATE.CREATE_CLUSTER]({ + metadataMaxAge, + allowAutoTopicCreation, + maxInFlightRequests, + isolationLevel, + instrumentationEmitter, + }) + + return createConsumer({ + retry: { ...this[PRIVATE.CLUSTER_RETRY], ...retry }, + logger: this[PRIVATE.LOGGER], + cluster, + groupId, + partitionAssigners, + sessionTimeout, + rebalanceTimeout, + heartbeatInterval, + maxBytesPerPartition, + minBytes, + maxBytes, + maxWaitTimeInMs, + isolationLevel, + instrumentationEmitter, + rackId, + metadataMaxAge, + }) + } + + /** + * @public + */ + admin({ retry } = {}) { + const instrumentationEmitter = new InstrumentationEventEmitter() + const cluster = this[PRIVATE.CREATE_CLUSTER]({ + allowAutoTopicCreation: false, + instrumentationEmitter, + }) + + return createAdmin({ + retry: { ...this[PRIVATE.CLUSTER_RETRY], ...retry }, + logger: this[PRIVATE.LOGGER], + instrumentationEmitter, + cluster, + }) + } + + /** + * @public + */ + logger() { + return this[PRIVATE.LOGGER] + } +} diff --git a/node_modules/kafkajs/src/instrumentation/emitter.js b/node_modules/kafkajs/src/instrumentation/emitter.js new file mode 100644 index 0000000..a9090a2 --- /dev/null +++ b/node_modules/kafkajs/src/instrumentation/emitter.js @@ -0,0 +1,34 @@ +const { EventEmitter } = require('events') +const InstrumentationEvent = require('./event') +const { KafkaJSError } = require('../errors') + +module.exports = class InstrumentationEventEmitter { + constructor() { + this.emitter = new EventEmitter() + } + + /** + * @param {string} eventName + * @param {Object} payload + */ + emit(eventName, payload) { + if (!eventName) { + throw new KafkaJSError('Invalid event name', { retriable: false }) + } + + if (this.emitter.listenerCount(eventName) > 0) { + const event = new InstrumentationEvent(eventName, payload) + this.emitter.emit(eventName, event) + } + } + + /** + * @param {string} eventName + * @param {(...args: any[]) => void} listener + * @returns {import("../../types").RemoveInstrumentationEventListener} removeListener + */ + addListener(eventName, listener) { + this.emitter.addListener(eventName, listener) + return () => this.emitter.removeListener(eventName, listener) + } +} diff --git a/node_modules/kafkajs/src/instrumentation/event.js b/node_modules/kafkajs/src/instrumentation/event.js new file mode 100644 index 0000000..711f9c4 --- /dev/null +++ b/node_modules/kafkajs/src/instrumentation/event.js @@ -0,0 +1,23 @@ +let id = 0 +const nextId = () => { + if (id === Number.MAX_VALUE) { + id = 0 + } + + return id++ +} + +class InstrumentationEvent { + /** + * @param {String} type + * @param {Object} payload + */ + constructor(type, payload) { + this.id = nextId() + this.type = type + this.timestamp = Date.now() + this.payload = payload + } +} + +module.exports = InstrumentationEvent diff --git a/node_modules/kafkajs/src/instrumentation/eventType.js b/node_modules/kafkajs/src/instrumentation/eventType.js new file mode 100644 index 0000000..e842ba8 --- /dev/null +++ b/node_modules/kafkajs/src/instrumentation/eventType.js @@ -0,0 +1,2 @@ +/** @type {(namespace: T1) => (type: T2) => `${T1}.${T2}`} */ +module.exports = namespace => type => `${namespace}.${type}` diff --git a/node_modules/kafkajs/src/loggers/console.js b/node_modules/kafkajs/src/loggers/console.js new file mode 100644 index 0000000..64dc18c --- /dev/null +++ b/node_modules/kafkajs/src/loggers/console.js @@ -0,0 +1,21 @@ +const { LEVELS: logLevel } = require('./index') + +module.exports = () => ({ namespace, level, label, log }) => { + const prefix = namespace ? `[${namespace}] ` : '' + const message = JSON.stringify( + Object.assign({ level: label }, log, { + message: `${prefix}${log.message}`, + }) + ) + + switch (level) { + case logLevel.INFO: + return console.info(message) + case logLevel.ERROR: + return console.error(message) + case logLevel.WARN: + return console.warn(message) + case logLevel.DEBUG: + return console.log(message) + } +} diff --git a/node_modules/kafkajs/src/loggers/index.js b/node_modules/kafkajs/src/loggers/index.js new file mode 100644 index 0000000..562b055 --- /dev/null +++ b/node_modules/kafkajs/src/loggers/index.js @@ -0,0 +1,68 @@ +const { assign } = Object + +const LEVELS = { + NOTHING: 0, + ERROR: 1, + WARN: 2, + INFO: 4, + DEBUG: 5, +} + +const createLevel = (label, level, currentLevel, namespace, logFunction) => ( + message, + extra = {} +) => { + if (level > currentLevel()) return + logFunction({ + namespace, + level, + label, + log: assign( + { + timestamp: new Date().toISOString(), + logger: 'kafkajs', + message, + }, + extra + ), + }) +} + +const evaluateLogLevel = logLevel => { + const envLogLevel = (process.env.KAFKAJS_LOG_LEVEL || '').toUpperCase() + return LEVELS[envLogLevel] == null ? logLevel : LEVELS[envLogLevel] +} + +const createLogger = ({ level = LEVELS.INFO, logCreator } = {}) => { + let logLevel = evaluateLogLevel(level) + const logFunction = logCreator(logLevel) + + const createNamespace = (namespace, logLevel = null) => { + const namespaceLogLevel = evaluateLogLevel(logLevel) + return createLogFunctions(namespace, namespaceLogLevel) + } + + const createLogFunctions = (namespace, namespaceLogLevel = null) => { + const currentLogLevel = () => (namespaceLogLevel == null ? logLevel : namespaceLogLevel) + const logger = { + info: createLevel('INFO', LEVELS.INFO, currentLogLevel, namespace, logFunction), + error: createLevel('ERROR', LEVELS.ERROR, currentLogLevel, namespace, logFunction), + warn: createLevel('WARN', LEVELS.WARN, currentLogLevel, namespace, logFunction), + debug: createLevel('DEBUG', LEVELS.DEBUG, currentLogLevel, namespace, logFunction), + } + + return assign(logger, { + namespace: createNamespace, + setLogLevel: newLevel => { + logLevel = newLevel + }, + }) + } + + return createLogFunctions() +} + +module.exports = { + LEVELS, + createLogger, +} diff --git a/node_modules/kafkajs/src/network/connection.js b/node_modules/kafkajs/src/network/connection.js new file mode 100644 index 0000000..dded584 --- /dev/null +++ b/node_modules/kafkajs/src/network/connection.js @@ -0,0 +1,543 @@ +const createSocket = require('./socket') +const createRequest = require('../protocol/request') +const Decoder = require('../protocol/decoder') +const { KafkaJSConnectionError, KafkaJSConnectionClosedError } = require('../errors') +const { INT_32_MAX_VALUE } = require('../constants') +const getEnv = require('../env') +const RequestQueue = require('./requestQueue') +const { CONNECTION_STATUS, CONNECTED_STATUS } = require('./connectionStatus') +const sharedPromiseTo = require('../utils/sharedPromiseTo') +const Long = require('../utils/long') +const SASLAuthenticator = require('../broker/saslAuthenticator') +const apiKeys = require('../protocol/requests/apiKeys') + +const requestInfo = ({ apiName, apiKey, apiVersion }) => + `${apiName}(key: ${apiKey}, version: ${apiVersion})` + +/** + * @param request - request from protocol + * @returns {boolean} + */ +const isAuthenticatedRequest = request => { + return ![apiKeys.ApiVersions, apiKeys.SaslHandshake, apiKeys.SaslAuthenticate].includes( + request.apiKey + ) +} + +const PRIVATE = { + SHOULD_REAUTHENTICATE: Symbol('private:Connection:shouldReauthenticate'), + AUTHENTICATE: Symbol('private:Connection:authenticate'), +} + +module.exports = class Connection { + /** + * @param {Object} options + * @param {string} options.host + * @param {number} options.port + * @param {import("../../types").Logger} options.logger + * @param {import("../../types").ISocketFactory} options.socketFactory + * @param {string} [options.clientId='kafkajs'] + * @param {number} options.requestTimeout The maximum amount of time the client will wait for the response of a request, + * in milliseconds + * @param {string} [options.rack=null] + * @param {Object} [options.ssl=null] Options for the TLS Secure Context. It accepts all options, + * usually "cert", "key" and "ca". More information at + * https://nodejs.org/api/tls.html#tls_tls_createsecurecontext_options + * @param {Object} [options.sasl=null] Attributes used for SASL authentication. Options based on the + * key "mechanism". Connection is not actively using the SASL attributes + * but acting as a data object for this information + * @param {number} [options.reauthenticationThreshold=10000] + * @param {number} options.connectionTimeout The connection timeout, in milliseconds + * @param {boolean} [options.enforceRequestTimeout] + * @param {number} [options.maxInFlightRequests=null] The maximum number of unacknowledged requests on a connection before + * enqueuing + * @param {import("../instrumentation/emitter")} [options.instrumentationEmitter=null] + */ + constructor({ + host, + port, + logger, + socketFactory, + requestTimeout, + reauthenticationThreshold = 10000, + rack = null, + ssl = null, + sasl = null, + clientId = 'kafkajs', + connectionTimeout, + enforceRequestTimeout = true, + maxInFlightRequests = null, + instrumentationEmitter = null, + }) { + this.host = host + this.port = port + this.rack = rack + this.clientId = clientId + this.broker = `${this.host}:${this.port}` + this.logger = logger.namespace('Connection') + + this.socketFactory = socketFactory + this.ssl = ssl + this.sasl = sasl + + this.requestTimeout = requestTimeout + this.connectionTimeout = connectionTimeout + this.reauthenticationThreshold = reauthenticationThreshold + + this.bytesBuffered = 0 + this.bytesNeeded = Decoder.int32Size() + this.chunks = [] + + this.connectionStatus = CONNECTION_STATUS.DISCONNECTED + this.correlationId = 0 + this.requestQueue = new RequestQueue({ + instrumentationEmitter, + maxInFlightRequests, + requestTimeout, + enforceRequestTimeout, + clientId, + broker: this.broker, + logger: logger.namespace('RequestQueue'), + isConnected: () => this.isConnected(), + }) + + this.versions = null + + this.authHandlers = null + this.authExpectResponse = false + + const log = level => (message, extra = {}) => { + const logFn = this.logger[level] + logFn(message, { broker: this.broker, clientId, ...extra }) + } + + this.logDebug = log('debug') + this.logError = log('error') + + const env = getEnv() + this.shouldLogBuffers = env.KAFKAJS_DEBUG_PROTOCOL_BUFFERS === '1' + this.shouldLogFetchBuffer = + this.shouldLogBuffers && env.KAFKAJS_DEBUG_EXTENDED_PROTOCOL_BUFFERS === '1' + + this.authenticatedAt = null + this.sessionLifetime = Long.ZERO + this.supportAuthenticationProtocol = null + + /** + * @private + * @returns {Promise} + */ + this[PRIVATE.AUTHENTICATE] = sharedPromiseTo(async () => { + if (this.sasl && !this.isAuthenticated()) { + const authenticator = new SASLAuthenticator( + this, + this.logger, + this.versions, + this.supportAuthenticationProtocol + ) + + await authenticator.authenticate() + this.authenticatedAt = process.hrtime() + this.sessionLifetime = Long.fromValue(authenticator.sessionLifetime) + } + }) + } + + getSupportAuthenticationProtocol() { + return this.supportAuthenticationProtocol + } + + setSupportAuthenticationProtocol(isSupported) { + this.supportAuthenticationProtocol = isSupported + } + + setVersions(versions) { + this.versions = versions + } + + isConnected() { + return CONNECTED_STATUS.includes(this.connectionStatus) + } + + /** + * @public + * @returns {Promise} + */ + connect() { + return new Promise((resolve, reject) => { + if (this.isConnected()) { + return resolve(true) + } + + this.authenticatedAt = null + + let timeoutId + + const onConnect = () => { + clearTimeout(timeoutId) + this.connectionStatus = CONNECTION_STATUS.CONNECTED + this.requestQueue.scheduleRequestTimeoutCheck() + resolve(true) + } + + const onData = data => { + this.processData(data) + } + + const onEnd = async () => { + clearTimeout(timeoutId) + + const wasConnected = this.isConnected() + + if (this.authHandlers) { + this.authHandlers.onError() + } else if (wasConnected) { + this.logDebug('Kafka server has closed connection') + this.rejectRequests( + new KafkaJSConnectionClosedError('Closed connection', { + host: this.host, + port: this.port, + }) + ) + } + + await this.disconnect() + } + + const onError = async e => { + clearTimeout(timeoutId) + + const error = new KafkaJSConnectionError(`Connection error: ${e.message}`, { + broker: `${this.host}:${this.port}`, + code: e.code, + }) + + this.logError(error.message, { stack: e.stack }) + this.rejectRequests(error) + await this.disconnect() + + reject(error) + } + + const onTimeout = async () => { + const error = new KafkaJSConnectionError('Connection timeout', { + broker: `${this.host}:${this.port}`, + }) + + this.logError(error.message) + this.rejectRequests(error) + await this.disconnect() + reject(error) + } + + this.logDebug(`Connecting`, { + ssl: !!this.ssl, + sasl: !!this.sasl, + }) + + try { + timeoutId = setTimeout(onTimeout, this.connectionTimeout) + this.socket = createSocket({ + socketFactory: this.socketFactory, + host: this.host, + port: this.port, + ssl: this.ssl, + onConnect, + onData, + onEnd, + onError, + onTimeout, + }) + } catch (e) { + clearTimeout(timeoutId) + reject( + new KafkaJSConnectionError(`Failed to connect: ${e.message}`, { + broker: `${this.host}:${this.port}`, + }) + ) + } + }) + } + + /** + * @public + * @returns {Promise} + */ + async disconnect() { + this.authenticatedAt = null + this.connectionStatus = CONNECTION_STATUS.DISCONNECTING + this.logDebug('disconnecting...') + + await this.requestQueue.waitForPendingRequests() + this.requestQueue.destroy() + + if (this.socket) { + this.socket.end() + this.socket.unref() + } + + this.connectionStatus = CONNECTION_STATUS.DISCONNECTED + this.logDebug('disconnected') + return true + } + + /** + * @public + * @returns {boolean} + */ + isAuthenticated() { + return this.authenticatedAt != null && !this[PRIVATE.SHOULD_REAUTHENTICATE]() + } + + /*** + * @private + */ + [PRIVATE.SHOULD_REAUTHENTICATE]() { + if (this.sessionLifetime.equals(Long.ZERO)) { + return false + } + + if (this.authenticatedAt == null) { + return true + } + + const [secondsSince, remainingNanosSince] = process.hrtime(this.authenticatedAt) + const millisSince = Long.fromValue(secondsSince) + .multiply(1000) + .add(Long.fromValue(remainingNanosSince).divide(1000000)) + + const reauthenticateAt = millisSince.add(this.reauthenticationThreshold) + return reauthenticateAt.greaterThanOrEqual(this.sessionLifetime) + } + + /** @public */ + async authenticate() { + await this[PRIVATE.AUTHENTICATE]() + } + + /** + * @public + * @returns {Promise} + */ + sendAuthRequest({ request, response }) { + this.authExpectResponse = !!response + + /** + * TODO: rewrite removing the async promise executor + */ + + /* eslint-disable no-async-promise-executor */ + return new Promise(async (resolve, reject) => { + this.authHandlers = { + onSuccess: rawData => { + this.authHandlers = null + this.authExpectResponse = false + + response + .decode(rawData) + .then(data => response.parse(data)) + .then(resolve) + .catch(reject) + }, + onError: () => { + this.authHandlers = null + this.authExpectResponse = false + + reject( + new KafkaJSConnectionError('Connection closed by the server', { + broker: `${this.host}:${this.port}`, + }) + ) + }, + } + + try { + const requestPayload = await request.encode() + + this.failIfNotConnected() + this.socket.write(requestPayload, 'binary') + } catch (e) { + reject(e) + } + }) + } + + /** + * @public + * @param {object} protocol + * @param {object} protocol.request It is defined by the protocol and consists of an object with "apiKey", + * "apiVersion", "apiName" and an "encode" function. The encode function + * must return an instance of Encoder + * + * @param {object} protocol.response It is defined by the protocol and consists of an object with two functions: + * "decode" and "parse" + * + * @param {number} [protocol.requestTimeout=null] Override for the default requestTimeout + * @param {boolean} [protocol.logResponseError=true] Whether to log errors + * @returns {Promise} where data is the return of "response#parse" + */ + async send({ request, response, requestTimeout = null, logResponseError = true }) { + if (!this.isAuthenticated() && isAuthenticatedRequest(request)) { + await this[PRIVATE.AUTHENTICATE]() + } + + this.failIfNotConnected() + + const expectResponse = !request.expectResponse || request.expectResponse() + const sendRequest = async () => { + const { clientId } = this + const correlationId = this.nextCorrelationId() + + const requestPayload = await createRequest({ request, correlationId, clientId }) + const { apiKey, apiName, apiVersion } = request + this.logDebug(`Request ${requestInfo(request)}`, { + correlationId, + expectResponse, + size: Buffer.byteLength(requestPayload.buffer), + }) + + return new Promise((resolve, reject) => { + try { + this.failIfNotConnected() + const entry = { apiKey, apiName, apiVersion, correlationId, resolve, reject } + + this.requestQueue.push({ + entry, + expectResponse, + requestTimeout, + sendRequest: () => { + this.socket.write(requestPayload.buffer, 'binary') + }, + }) + } catch (e) { + reject(e) + } + }) + } + + const { correlationId, size, entry, payload } = await sendRequest() + + if (!expectResponse) { + return + } + + try { + const payloadDecoded = await response.decode(payload) + + /** + * @see KIP-219 + * If the response indicates that the client-side needs to throttle, do that. + */ + this.requestQueue.maybeThrottle(payloadDecoded.clientSideThrottleTime) + + const data = await response.parse(payloadDecoded) + const isFetchApi = entry.apiName === 'Fetch' + this.logDebug(`Response ${requestInfo(entry)}`, { + correlationId, + size, + data: isFetchApi && !this.shouldLogFetchBuffer ? '[filtered]' : data, + }) + + return data + } catch (e) { + if (logResponseError) { + this.logError(`Response ${requestInfo(entry)}`, { + error: e.message, + correlationId, + size, + }) + } + + const isBuffer = Buffer.isBuffer(payload) + this.logDebug(`Response ${requestInfo(entry)}`, { + error: e.message, + correlationId, + payload: + isBuffer && !this.shouldLogBuffers ? { type: 'Buffer', data: '[filtered]' } : payload, + }) + + throw e + } + } + + /** + * @private + */ + failIfNotConnected() { + if (!this.isConnected()) { + throw new KafkaJSConnectionError('Not connected', { + broker: `${this.host}:${this.port}`, + }) + } + } + + /** + * @private + */ + nextCorrelationId() { + if (this.correlationId >= INT_32_MAX_VALUE) { + this.correlationId = 0 + } + + return this.correlationId++ + } + + /** + * @private + */ + processData(rawData) { + if (this.authHandlers && !this.authExpectResponse) { + return this.authHandlers.onSuccess(rawData) + } + + // Accumulate the new chunk + this.chunks.push(rawData) + this.bytesBuffered += Buffer.byteLength(rawData) + + // Process data if there are enough bytes to read the expected response size, + // otherwise keep buffering + while (this.bytesNeeded <= this.bytesBuffered) { + const buffer = this.chunks.length > 1 ? Buffer.concat(this.chunks) : this.chunks[0] + const decoder = new Decoder(buffer) + const expectedResponseSize = decoder.readInt32() + + // Return early if not enough bytes to read the full response + if (!decoder.canReadBytes(expectedResponseSize)) { + this.chunks = [buffer] + this.bytesBuffered = Buffer.byteLength(buffer) + this.bytesNeeded = Decoder.int32Size() + expectedResponseSize + return + } + + const response = new Decoder(decoder.readBytes(expectedResponseSize)) + + // Reset the buffered chunks as the rest of the bytes + const remainderBuffer = decoder.readAll() + this.chunks = [remainderBuffer] + this.bytesBuffered = Buffer.byteLength(remainderBuffer) + this.bytesNeeded = Decoder.int32Size() + + if (this.authHandlers) { + const rawResponseSize = Decoder.int32Size() + expectedResponseSize + const rawResponseBuffer = buffer.slice(0, rawResponseSize) + return this.authHandlers.onSuccess(rawResponseBuffer) + } + + const correlationId = response.readInt32() + const payload = response.readAll() + + this.requestQueue.fulfillRequest({ + size: expectedResponseSize, + correlationId, + payload, + }) + } + } + + /** + * @private + */ + rejectRequests(error) { + this.requestQueue.rejectAll(error) + } +} diff --git a/node_modules/kafkajs/src/network/connectionPool.js b/node_modules/kafkajs/src/network/connectionPool.js new file mode 100644 index 0000000..ebb77b3 --- /dev/null +++ b/node_modules/kafkajs/src/network/connectionPool.js @@ -0,0 +1,65 @@ +const apiKeys = require('../protocol/requests/apiKeys') +const Connection = require('./connection') + +module.exports = class ConnectionPool { + /** + * @param {ConstructorParameters[0]} options + */ + constructor(options) { + this.logger = options.logger.namespace('ConnectionPool') + this.connectionTimeout = options.connectionTimeout + this.host = options.host + this.port = options.port + this.rack = options.rack + this.ssl = options.ssl + this.sasl = options.sasl + this.clientId = options.clientId + this.socketFactory = options.socketFactory + + this.pool = new Array(2).fill().map(() => new Connection(options)) + } + + isConnected() { + return this.pool.some(c => c.isConnected()) + } + + isAuthenticated() { + return this.pool.some(c => c.isAuthenticated()) + } + + setSupportAuthenticationProtocol(isSupported) { + this.map(c => c.setSupportAuthenticationProtocol(isSupported)) + } + + setVersions(versions) { + this.map(c => c.setVersions(versions)) + } + + map(callback) { + return this.pool.map(c => callback(c)) + } + + async send(protocolRequest) { + const connection = await this.getConnectionByRequest(protocolRequest) + return connection.send(protocolRequest) + } + + getConnectionByRequest({ request: { apiKey } }) { + const index = { [apiKeys.Fetch]: 1 }[apiKey] || 0 + return this.getConnection(index) + } + + async getConnection(index = 0) { + const connection = this.pool[index] + + if (!connection.isConnected()) { + await connection.connect() + } + + return connection + } + + async destroy() { + await Promise.all(this.map(c => c.disconnect())) + } +} diff --git a/node_modules/kafkajs/src/network/connectionStatus.js b/node_modules/kafkajs/src/network/connectionStatus.js new file mode 100644 index 0000000..65cf90f --- /dev/null +++ b/node_modules/kafkajs/src/network/connectionStatus.js @@ -0,0 +1,12 @@ +const CONNECTION_STATUS = { + CONNECTED: 'connected', + DISCONNECTING: 'disconnecting', + DISCONNECTED: 'disconnected', +} + +const CONNECTED_STATUS = [CONNECTION_STATUS.CONNECTED, CONNECTION_STATUS.DISCONNECTING] + +module.exports = { + CONNECTION_STATUS, + CONNECTED_STATUS, +} diff --git a/node_modules/kafkajs/src/network/instrumentationEvents.js b/node_modules/kafkajs/src/network/instrumentationEvents.js new file mode 100644 index 0000000..3d7eac7 --- /dev/null +++ b/node_modules/kafkajs/src/network/instrumentationEvents.js @@ -0,0 +1,8 @@ +const InstrumentationEventType = require('../instrumentation/eventType') +const eventType = InstrumentationEventType('network') + +module.exports = { + NETWORK_REQUEST: eventType('request'), + NETWORK_REQUEST_TIMEOUT: eventType('request_timeout'), + NETWORK_REQUEST_QUEUE_SIZE: eventType('request_queue_size'), +} diff --git a/node_modules/kafkajs/src/network/requestQueue/index.js b/node_modules/kafkajs/src/network/requestQueue/index.js new file mode 100644 index 0000000..eee14d6 --- /dev/null +++ b/node_modules/kafkajs/src/network/requestQueue/index.js @@ -0,0 +1,323 @@ +const { EventEmitter } = require('events') +const SocketRequest = require('./socketRequest') +const events = require('../instrumentationEvents') +const { KafkaJSInvariantViolation } = require('../../errors') + +const PRIVATE = { + EMIT_QUEUE_SIZE_EVENT: Symbol('private:RequestQueue:emitQueueSizeEvent'), + EMIT_REQUEST_QUEUE_EMPTY: Symbol('private:RequestQueue:emitQueueEmpty'), +} + +const REQUEST_QUEUE_EMPTY = 'requestQueueEmpty' +const CHECK_PENDING_REQUESTS_INTERVAL = 10 + +module.exports = class RequestQueue extends EventEmitter { + /** + * @param {Object} options + * @param {number} options.maxInFlightRequests + * @param {number} options.requestTimeout + * @param {boolean} options.enforceRequestTimeout + * @param {string} options.clientId + * @param {string} options.broker + * @param {import("../../../types").Logger} options.logger + * @param {import("../../instrumentation/emitter")} [options.instrumentationEmitter=null] + * @param {() => boolean} [options.isConnected] + */ + constructor({ + instrumentationEmitter = null, + maxInFlightRequests, + requestTimeout, + enforceRequestTimeout, + clientId, + broker, + logger, + isConnected = () => true, + }) { + super() + this.instrumentationEmitter = instrumentationEmitter + this.maxInFlightRequests = maxInFlightRequests + this.requestTimeout = requestTimeout + this.enforceRequestTimeout = enforceRequestTimeout + this.clientId = clientId + this.broker = broker + this.logger = logger + this.isConnected = isConnected + + this.inflight = new Map() + this.pending = [] + + /** + * Until when this request queue is throttled and shouldn't send requests + * + * The value represents the timestamp of the end of the throttling in ms-since-epoch. If the value + * is smaller than the current timestamp no throttling is active. + * + * @type {number} + */ + this.throttledUntil = -1 + + /** + * Timeout id if we have scheduled a check for pending requests due to client-side throttling + * + * @type {null|NodeJS.Timeout} + */ + this.throttleCheckTimeoutId = null + + this[PRIVATE.EMIT_REQUEST_QUEUE_EMPTY] = () => { + if (this.pending.length === 0 && this.inflight.size === 0) { + this.emit(REQUEST_QUEUE_EMPTY) + } + } + + this[PRIVATE.EMIT_QUEUE_SIZE_EVENT] = () => { + instrumentationEmitter && + instrumentationEmitter.emit(events.NETWORK_REQUEST_QUEUE_SIZE, { + broker: this.broker, + clientId: this.clientId, + queueSize: this.pending.length, + }) + + this[PRIVATE.EMIT_REQUEST_QUEUE_EMPTY]() + } + } + + /** + * @public + */ + scheduleRequestTimeoutCheck() { + if (this.enforceRequestTimeout) { + this.destroy() + + this.requestTimeoutIntervalId = setInterval(() => { + this.inflight.forEach(request => { + if (Date.now() - request.sentAt > request.requestTimeout) { + request.timeoutRequest() + } + }) + + if (!this.isConnected()) { + this.destroy() + } + }, Math.min(this.requestTimeout, 100)) + } + } + + maybeThrottle(clientSideThrottleTime) { + if (clientSideThrottleTime !== null && clientSideThrottleTime > 0) { + this.logger.debug(`Client side throttling in effect for ${clientSideThrottleTime}ms`) + const minimumThrottledUntil = Date.now() + clientSideThrottleTime + this.throttledUntil = Math.max(minimumThrottledUntil, this.throttledUntil) + } + } + + createSocketRequest(pushedRequest) { + const { correlationId } = pushedRequest.entry + const defaultRequestTimeout = this.requestTimeout + const customRequestTimeout = pushedRequest.requestTimeout + + // Some protocol requests have custom request timeouts (e.g JoinGroup, Fetch, etc). The custom + // timeouts are influenced by user configurations, which can be lower than the default requestTimeout + const requestTimeout = Math.max(defaultRequestTimeout, customRequestTimeout || 0) + + const socketRequest = new SocketRequest({ + entry: pushedRequest.entry, + expectResponse: pushedRequest.expectResponse, + broker: this.broker, + clientId: this.clientId, + instrumentationEmitter: this.instrumentationEmitter, + requestTimeout, + send: () => { + if (this.inflight.has(correlationId)) { + throw new KafkaJSInvariantViolation('Correlation id already exists') + } + this.inflight.set(correlationId, socketRequest) + pushedRequest.sendRequest() + }, + timeout: () => { + this.inflight.delete(correlationId) + this.checkPendingRequests() + // Try to emit REQUEST_QUEUE_EMPTY. Otherwise, waitForPendingRequests may stuck forever + this[PRIVATE.EMIT_REQUEST_QUEUE_EMPTY]() + }, + }) + + return socketRequest + } + + /** + * @typedef {Object} PushedRequest + * @property {import("./socketRequest").RequestEntry} entry + * @property {boolean} expectResponse + * @property {Function} sendRequest + * @property {number} [requestTimeout] + * + * @public + * @param {PushedRequest} pushedRequest + */ + push(pushedRequest) { + const { correlationId } = pushedRequest.entry + const socketRequest = this.createSocketRequest(pushedRequest) + + if (this.canSendSocketRequestImmediately()) { + this.sendSocketRequest(socketRequest) + return + } + + this.pending.push(socketRequest) + this.scheduleCheckPendingRequests() + + this.logger.debug(`Request enqueued`, { + clientId: this.clientId, + broker: this.broker, + correlationId, + }) + + this[PRIVATE.EMIT_QUEUE_SIZE_EVENT]() + } + + /** + * @param {SocketRequest} socketRequest + */ + sendSocketRequest(socketRequest) { + socketRequest.send() + + if (!socketRequest.expectResponse) { + this.logger.debug(`Request does not expect a response, resolving immediately`, { + clientId: this.clientId, + broker: this.broker, + correlationId: socketRequest.correlationId, + }) + + this.inflight.delete(socketRequest.correlationId) + socketRequest.completed({ size: 0, payload: null }) + } + } + + /** + * @public + * @param {object} response + * @param {number} response.correlationId + * @param {Buffer} response.payload + * @param {number} response.size + */ + fulfillRequest({ correlationId, payload, size }) { + const socketRequest = this.inflight.get(correlationId) + this.inflight.delete(correlationId) + this.checkPendingRequests() + + if (socketRequest) { + socketRequest.completed({ size, payload }) + } else { + this.logger.warn(`Response without match`, { + clientId: this.clientId, + broker: this.broker, + correlationId, + }) + } + + this[PRIVATE.EMIT_REQUEST_QUEUE_EMPTY]() + } + + /** + * @public + * @param {Error} error + */ + rejectAll(error) { + const requests = [...this.inflight.values(), ...this.pending] + + for (const socketRequest of requests) { + socketRequest.rejected(error) + this.inflight.delete(socketRequest.correlationId) + } + + this.pending = [] + this.inflight.clear() + this[PRIVATE.EMIT_QUEUE_SIZE_EVENT]() + } + + /** + * @public + */ + waitForPendingRequests() { + return new Promise(resolve => { + if (this.pending.length === 0 && this.inflight.size === 0) { + return resolve() + } + + this.logger.debug('Waiting for pending requests', { + clientId: this.clientId, + broker: this.broker, + currentInflightRequests: this.inflight.size, + currentPendingQueueSize: this.pending.length, + }) + + this.once(REQUEST_QUEUE_EMPTY, () => resolve()) + }) + } + + /** + * @public + */ + destroy() { + clearInterval(this.requestTimeoutIntervalId) + clearTimeout(this.throttleCheckTimeoutId) + this.throttleCheckTimeoutId = null + } + + canSendSocketRequestImmediately() { + const shouldEnqueue = + (this.maxInFlightRequests != null && this.inflight.size >= this.maxInFlightRequests) || + this.throttledUntil > Date.now() + + return !shouldEnqueue + } + + /** + * Check and process pending requests either now or in the future + * + * This function will send out as many pending requests as possible taking throttling and + * in-flight limits into account. + */ + checkPendingRequests() { + while (this.pending.length > 0 && this.canSendSocketRequestImmediately()) { + const pendingRequest = this.pending.shift() // first in first out + this.sendSocketRequest(pendingRequest) + + this.logger.debug(`Consumed pending request`, { + clientId: this.clientId, + broker: this.broker, + correlationId: pendingRequest.correlationId, + pendingDuration: pendingRequest.pendingDuration, + currentPendingQueueSize: this.pending.length, + }) + + this[PRIVATE.EMIT_QUEUE_SIZE_EVENT]() + } + + this.scheduleCheckPendingRequests() + } + + /** + * Ensure that pending requests will be checked in the future + * + * If there is a client-side throttling in place this will ensure that we will check + * the pending request queue eventually. + */ + scheduleCheckPendingRequests() { + // If we're throttled: Schedule checkPendingRequests when the throttle + // should be resolved. If there is already something scheduled we assume that that + // will be fine, and potentially fix up a new timeout if needed at that time. + // Note that if we're merely "overloaded" by having too many inflight requests + // we will anyways check the queue when one of them gets fulfilled. + let scheduleAt = this.throttledUntil - Date.now() + if (!this.throttleCheckTimeoutId) { + if (this.pending.length > 0) { + scheduleAt = scheduleAt > 0 ? scheduleAt : CHECK_PENDING_REQUESTS_INTERVAL + } + this.throttleCheckTimeoutId = setTimeout(() => { + this.throttleCheckTimeoutId = null + this.checkPendingRequests() + }, scheduleAt) + } + } +} diff --git a/node_modules/kafkajs/src/network/requestQueue/socketRequest.js b/node_modules/kafkajs/src/network/requestQueue/socketRequest.js new file mode 100644 index 0000000..301f750 --- /dev/null +++ b/node_modules/kafkajs/src/network/requestQueue/socketRequest.js @@ -0,0 +1,168 @@ +const { KafkaJSRequestTimeoutError, KafkaJSNonRetriableError } = require('../../errors') +const events = require('../instrumentationEvents') + +const PRIVATE = { + STATE: Symbol('private:SocketRequest:state'), + EMIT_EVENT: Symbol('private:SocketRequest:emitEvent'), +} + +const REQUEST_STATE = { + PENDING: Symbol('PENDING'), + SENT: Symbol('SENT'), + COMPLETED: Symbol('COMPLETED'), + REJECTED: Symbol('REJECTED'), +} + +/** + * SocketRequest abstracts the life cycle of a socket request, making it easier to track + * request durations and to have individual timeouts per request. + * + * @typedef {Object} SocketRequest + * @property {number} createdAt + * @property {number} sentAt + * @property {number} pendingDuration + * @property {number} duration + * @property {number} requestTimeout + * @property {string} broker + * @property {string} clientId + * @property {RequestEntry} entry + * @property {boolean} expectResponse + * @property {Function} send + * @property {Function} timeout + * + * @typedef {Object} RequestEntry + * @property {string} apiKey + * @property {string} apiName + * @property {number} apiVersion + * @property {number} correlationId + * @property {Function} resolve + * @property {Function} reject + */ +module.exports = class SocketRequest { + /** + * @param {Object} options + * @param {number} options.requestTimeout + * @param {string} options.broker - e.g: 127.0.0.1:9092 + * @param {string} options.clientId + * @param {RequestEntry} options.entry + * @param {boolean} options.expectResponse + * @param {Function} options.send + * @param {() => void} options.timeout + * @param {import("../../instrumentation/emitter")} [options.instrumentationEmitter=null] + */ + constructor({ + requestTimeout, + broker, + clientId, + entry, + expectResponse, + send, + timeout, + instrumentationEmitter = null, + }) { + this.createdAt = Date.now() + this.requestTimeout = requestTimeout + this.broker = broker + this.clientId = clientId + this.entry = entry + this.correlationId = entry.correlationId + this.expectResponse = expectResponse + this.sendRequest = send + this.timeoutHandler = timeout + + this.sentAt = null + this.duration = null + this.pendingDuration = null + + this[PRIVATE.STATE] = REQUEST_STATE.PENDING + this[PRIVATE.EMIT_EVENT] = (eventName, payload) => + instrumentationEmitter && instrumentationEmitter.emit(eventName, payload) + } + + send() { + this.throwIfInvalidState({ + accepted: [REQUEST_STATE.PENDING], + next: REQUEST_STATE.SENT, + }) + + this.sendRequest() + this.sentAt = Date.now() + this.pendingDuration = this.sentAt - this.createdAt + this[PRIVATE.STATE] = REQUEST_STATE.SENT + } + + timeoutRequest() { + const { apiName, apiKey, apiVersion } = this.entry + const requestInfo = `${apiName}(key: ${apiKey}, version: ${apiVersion})` + const eventData = { + broker: this.broker, + clientId: this.clientId, + correlationId: this.correlationId, + createdAt: this.createdAt, + sentAt: this.sentAt, + pendingDuration: this.pendingDuration, + } + + this.timeoutHandler() + this.rejected(new KafkaJSRequestTimeoutError(`Request ${requestInfo} timed out`, eventData)) + this[PRIVATE.EMIT_EVENT](events.NETWORK_REQUEST_TIMEOUT, { + ...eventData, + apiName, + apiKey, + apiVersion, + }) + } + + completed({ size, payload }) { + this.throwIfInvalidState({ + accepted: [REQUEST_STATE.SENT], + next: REQUEST_STATE.COMPLETED, + }) + + const { entry, correlationId, broker, clientId, createdAt, sentAt, pendingDuration } = this + + this[PRIVATE.STATE] = REQUEST_STATE.COMPLETED + this.duration = Date.now() - this.sentAt + entry.resolve({ correlationId, entry, size, payload }) + + this[PRIVATE.EMIT_EVENT](events.NETWORK_REQUEST, { + broker, + clientId, + correlationId, + size, + createdAt, + sentAt, + pendingDuration, + duration: this.duration, + apiName: entry.apiName, + apiKey: entry.apiKey, + apiVersion: entry.apiVersion, + }) + } + + rejected(error) { + this.throwIfInvalidState({ + accepted: [REQUEST_STATE.PENDING, REQUEST_STATE.SENT], + next: REQUEST_STATE.REJECTED, + }) + + this[PRIVATE.STATE] = REQUEST_STATE.REJECTED + this.duration = Date.now() - this.sentAt + this.entry.reject(error) + } + + /** + * @private + */ + throwIfInvalidState({ accepted, next }) { + if (accepted.includes(this[PRIVATE.STATE])) { + return + } + + const current = this[PRIVATE.STATE].toString() + + throw new KafkaJSNonRetriableError( + `Invalid state, can't transition from ${current} to ${next.toString()}` + ) + } +} diff --git a/node_modules/kafkajs/src/network/socket.js b/node_modules/kafkajs/src/network/socket.js new file mode 100644 index 0000000..6297669 --- /dev/null +++ b/node_modules/kafkajs/src/network/socket.js @@ -0,0 +1,32 @@ +/** + * @param {Object} options + * @param {import("../../types").ISocketFactory} options.socketFactory + * @param {string} options.host + * @param {number} options.port + * @param {Object} options.ssl + * @param {() => void} options.onConnect + * @param {(data: Buffer) => void} options.onData + * @param {() => void} options.onEnd + * @param {(err: Error) => void} options.onError + * @param {() => void} options.onTimeout + */ +module.exports = ({ + socketFactory, + host, + port, + ssl, + onConnect, + onData, + onEnd, + onError, + onTimeout, +}) => { + const socket = socketFactory({ host, port, ssl, onConnect }) + + socket.on('data', onData) + socket.on('end', onEnd) + socket.on('error', onError) + socket.on('timeout', onTimeout) + + return socket +} diff --git a/node_modules/kafkajs/src/network/socketFactory.js b/node_modules/kafkajs/src/network/socketFactory.js new file mode 100644 index 0000000..97b208f --- /dev/null +++ b/node_modules/kafkajs/src/network/socketFactory.js @@ -0,0 +1,22 @@ +const KEEP_ALIVE_DELAY = 60000 // in ms + +/** + * @returns {import("../../types").ISocketFactory} + */ +module.exports = () => { + const net = require('net') + const tls = require('tls') + + return ({ host, port, ssl, onConnect }) => { + const socket = ssl + ? tls.connect( + Object.assign({ host, port }, !net.isIP(host) ? { servername: host } : {}, ssl), + onConnect + ) + : net.connect({ host, port }, onConnect) + + socket.setKeepAlive(true, KEEP_ALIVE_DELAY) + + return socket + } +} diff --git a/node_modules/kafkajs/src/producer/createTopicData.js b/node_modules/kafkajs/src/producer/createTopicData.js new file mode 100644 index 0000000..d07d3b9 --- /dev/null +++ b/node_modules/kafkajs/src/producer/createTopicData.js @@ -0,0 +1,11 @@ +module.exports = topicDataForBroker => { + return topicDataForBroker.map( + ({ topic, partitions, messagesPerPartition, sequencePerPartition }) => ({ + topic, + partitions: partitions.map(partition => ({ + partition, + messages: messagesPerPartition[partition], + })), + }) + ) +} diff --git a/node_modules/kafkajs/src/producer/eosManager/index.js b/node_modules/kafkajs/src/producer/eosManager/index.js new file mode 100644 index 0000000..3010db7 --- /dev/null +++ b/node_modules/kafkajs/src/producer/eosManager/index.js @@ -0,0 +1,464 @@ +const createRetry = require('../../retry') +const Lock = require('../../utils/lock') +const { KafkaJSNonRetriableError } = require('../../errors') +const COORDINATOR_TYPES = require('../../protocol/coordinatorTypes') +const createStateMachine = require('./transactionStateMachine') +const { INT_32_MAX_VALUE } = require('../../constants') +const assert = require('assert') + +const STATES = require('./transactionStates') +const NO_PRODUCER_ID = -1 +const SEQUENCE_START = 0 +const INIT_PRODUCER_RETRIABLE_PROTOCOL_ERRORS = [ + 'NOT_COORDINATOR_FOR_GROUP', + 'GROUP_COORDINATOR_NOT_AVAILABLE', + 'GROUP_LOAD_IN_PROGRESS', + /** + * The producer might have crashed and never committed the transaction; retry the + * request so Kafka can abort the current transaction + * @see https://github.com/apache/kafka/blob/201da0542726472d954080d54bc585b111aaf86f/clients/src/main/java/org/apache/kafka/clients/producer/internals/TransactionManager.java#L1001-L1002 + */ + 'CONCURRENT_TRANSACTIONS', +] +const COMMIT_RETRIABLE_PROTOCOL_ERRORS = [ + 'UNKNOWN_TOPIC_OR_PARTITION', + 'COORDINATOR_LOAD_IN_PROGRESS', +] +const COMMIT_STALE_COORDINATOR_PROTOCOL_ERRORS = ['COORDINATOR_NOT_AVAILABLE', 'NOT_COORDINATOR'] + +/** + * @typedef {Object} EosManager + */ + +/** + * Manage behavior for an idempotent producer and transactions. + * + * @returns {EosManager} + */ +module.exports = ({ + logger, + cluster, + transactionTimeout = 60000, + transactional, + transactionalId, +}) => { + if (transactional && !transactionalId) { + throw new KafkaJSNonRetriableError('Cannot manage transactions without a transactionalId') + } + + const retrier = createRetry(cluster.retry) + + /** + * Current producer ID + */ + let producerId = NO_PRODUCER_ID + + /** + * Current producer epoch + */ + let producerEpoch = 0 + + /** + * Idempotent production requires that the producer track the sequence number of messages. + * + * Sequences are sent with every Record Batch and tracked per Topic-Partition + */ + let producerSequence = {} + + /** + * Idempotent production requires a mutex lock per broker to serialize requests with sequence number handling + */ + let brokerMutexLocks = {} + + /** + * Topic partitions already participating in the transaction + */ + let transactionTopicPartitions = {} + + /** + * Offsets have been added to the transaction + */ + let hasOffsetsAddedToTransaction = false + + const stateMachine = createStateMachine({ logger }) + stateMachine.on('transition', ({ to }) => { + if (to === STATES.READY) { + transactionTopicPartitions = {} + hasOffsetsAddedToTransaction = false + } + }) + + const findTransactionCoordinator = () => { + return cluster.findGroupCoordinator({ + groupId: transactionalId, + coordinatorType: COORDINATOR_TYPES.TRANSACTION, + }) + } + + const transactionalGuard = () => { + if (!transactional) { + throw new KafkaJSNonRetriableError('Method unavailable if non-transactional') + } + } + + /** + * A transaction is ongoing when offsets or partitions added to it + * + * @returns {boolean} + */ + const isOngoing = () => { + return ( + hasOffsetsAddedToTransaction || + Object.entries(transactionTopicPartitions).some(([, partitions]) => { + return Object.entries(partitions).some( + ([, isPartitionAddedToTransaction]) => isPartitionAddedToTransaction + ) + }) + ) + } + + const eosManager = stateMachine.createGuarded( + { + /** + * Get the current producer id + * @returns {number} + */ + getProducerId() { + return producerId + }, + + /** + * Get the current producer epoch + * @returns {number} + */ + getProducerEpoch() { + return producerEpoch + }, + + getTransactionalId() { + return transactionalId + }, + + /** + * Initialize the idempotent producer by making an `InitProducerId` request. + * Overwrites any existing state in this transaction manager + */ + async initProducerId() { + return retrier(async (bail, retryCount, retryTime) => { + try { + await cluster.refreshMetadataIfNecessary() + + // If non-transactional we can request the PID from any broker + const broker = await (transactional + ? findTransactionCoordinator() + : cluster.findControllerBroker()) + + const result = await broker.initProducerId({ + transactionalId: transactional ? transactionalId : undefined, + transactionTimeout, + }) + + stateMachine.transitionTo(STATES.READY) + producerId = result.producerId + producerEpoch = result.producerEpoch + producerSequence = {} + brokerMutexLocks = {} + + logger.debug('Initialized producer id & epoch', { producerId, producerEpoch }) + } catch (e) { + if (INIT_PRODUCER_RETRIABLE_PROTOCOL_ERRORS.includes(e.type)) { + if (e.type === 'CONCURRENT_TRANSACTIONS') { + logger.debug('There is an ongoing transaction on this transactionId, retrying', { + error: e.message, + stack: e.stack, + transactionalId, + retryCount, + retryTime, + }) + } + + throw e + } + + bail(e) + } + }) + }, + + /** + * Get the current sequence for a given Topic-Partition. Defaults to 0. + * + * @param {string} topic + * @param {string} partition + * @returns {number} + */ + getSequence(topic, partition) { + if (!eosManager.isInitialized()) { + return SEQUENCE_START + } + + producerSequence[topic] = producerSequence[topic] || {} + producerSequence[topic][partition] = producerSequence[topic][partition] || SEQUENCE_START + + return producerSequence[topic][partition] + }, + + /** + * Update the sequence for a given Topic-Partition. + * + * Do nothing if not yet initialized (not idempotent) + * @param {string} topic + * @param {string} partition + * @param {number} increment + */ + updateSequence(topic, partition, increment) { + if (!eosManager.isInitialized()) { + return + } + + const previous = eosManager.getSequence(topic, partition) + let sequence = previous + increment + + // Sequence is defined as Int32 in the Record Batch, + // so theoretically should need to rotate here + if (sequence >= INT_32_MAX_VALUE) { + logger.debug( + `Sequence for ${topic} ${partition} exceeds max value (${sequence}). Rotating to 0.` + ) + sequence = 0 + } + + producerSequence[topic][partition] = sequence + }, + + /** + * Begin a transaction + */ + beginTransaction() { + transactionalGuard() + stateMachine.transitionTo(STATES.TRANSACTING) + }, + + /** + * Add partitions to a transaction if they are not already marked as participating. + * + * Should be called prior to sending any messages during a transaction + * @param {TopicData[]} topicData + * + * @typedef {Object} TopicData + * @property {string} topic + * @property {object[]} partitions + * @property {number} partitions[].partition + */ + async addPartitionsToTransaction(topicData) { + transactionalGuard() + const newTopicPartitions = {} + + topicData.forEach(({ topic, partitions }) => { + transactionTopicPartitions[topic] = transactionTopicPartitions[topic] || {} + + partitions.forEach(({ partition }) => { + if (!transactionTopicPartitions[topic][partition]) { + newTopicPartitions[topic] = newTopicPartitions[topic] || [] + newTopicPartitions[topic].push(partition) + } + }) + }) + + const topics = Object.keys(newTopicPartitions).map(topic => ({ + topic, + partitions: newTopicPartitions[topic], + })) + + if (topics.length) { + const broker = await findTransactionCoordinator() + await broker.addPartitionsToTxn({ transactionalId, producerId, producerEpoch, topics }) + } + + topics.forEach(({ topic, partitions }) => { + partitions.forEach(partition => { + transactionTopicPartitions[topic][partition] = true + }) + }) + }, + + /** + * Commit the ongoing transaction + */ + async commit() { + transactionalGuard() + stateMachine.transitionTo(STATES.COMMITTING) + + if (!isOngoing()) { + logger.debug('No partitions or offsets registered, not sending EndTxn') + + stateMachine.transitionTo(STATES.READY) + return + } + + const broker = await findTransactionCoordinator() + await broker.endTxn({ + producerId, + producerEpoch, + transactionalId, + transactionResult: true, + }) + + stateMachine.transitionTo(STATES.READY) + }, + + /** + * Abort the ongoing transaction + */ + async abort() { + transactionalGuard() + stateMachine.transitionTo(STATES.ABORTING) + + if (!isOngoing()) { + logger.debug('No partitions or offsets registered, not sending EndTxn') + + stateMachine.transitionTo(STATES.READY) + return + } + + const broker = await findTransactionCoordinator() + await broker.endTxn({ + producerId, + producerEpoch, + transactionalId, + transactionResult: false, + }) + + stateMachine.transitionTo(STATES.READY) + }, + + /** + * Whether the producer id has already been initialized + */ + isInitialized() { + return producerId !== NO_PRODUCER_ID + }, + + isTransactional() { + return transactional + }, + + isInTransaction() { + return stateMachine.state() === STATES.TRANSACTING + }, + + async acquireBrokerLock(broker) { + if (this.isInitialized()) { + brokerMutexLocks[broker.nodeId] = + brokerMutexLocks[broker.nodeId] || new Lock({ timeout: 0xffff }) + await brokerMutexLocks[broker.nodeId].acquire() + } + }, + + releaseBrokerLock(broker) { + if (this.isInitialized()) brokerMutexLocks[broker.nodeId].release() + }, + + /** + * Mark the provided offsets as participating in the transaction for the given consumer group. + * + * This allows us to commit an offset as consumed only if the transaction passes. + * @param {string} consumerGroupId The unique group identifier + * @param {OffsetCommitTopic[]} topics The unique group identifier + * @returns {Promise} + * + * @typedef {Object} OffsetCommitTopic + * @property {string} topic + * @property {OffsetCommitTopicPartition[]} partitions + * + * @typedef {Object} OffsetCommitTopicPartition + * @property {number} partition + * @property {number} offset + */ + async sendOffsets({ consumerGroupId, topics }) { + assert(consumerGroupId, 'Missing consumerGroupId') + assert(topics, 'Missing offset topics') + + const transactionCoordinator = await findTransactionCoordinator() + + // Do we need to add offsets if we've already done so for this consumer group? + await transactionCoordinator.addOffsetsToTxn({ + transactionalId, + producerId, + producerEpoch, + groupId: consumerGroupId, + }) + + hasOffsetsAddedToTransaction = true + + let groupCoordinator = await cluster.findGroupCoordinator({ + groupId: consumerGroupId, + coordinatorType: COORDINATOR_TYPES.GROUP, + }) + + return retrier(async (bail, retryCount, retryTime) => { + try { + await groupCoordinator.txnOffsetCommit({ + transactionalId, + producerId, + producerEpoch, + groupId: consumerGroupId, + topics, + }) + } catch (e) { + if (COMMIT_RETRIABLE_PROTOCOL_ERRORS.includes(e.type)) { + logger.debug('Group coordinator is not ready yet, retrying', { + error: e.message, + stack: e.stack, + transactionalId, + retryCount, + retryTime, + }) + + throw e + } + + if ( + COMMIT_STALE_COORDINATOR_PROTOCOL_ERRORS.includes(e.type) || + e.code === 'ECONNREFUSED' + ) { + logger.debug( + 'Invalid group coordinator, finding new group coordinator and retrying', + { + error: e.message, + stack: e.stack, + transactionalId, + retryCount, + retryTime, + } + ) + + groupCoordinator = await cluster.findGroupCoordinator({ + groupId: consumerGroupId, + coordinatorType: COORDINATOR_TYPES.GROUP, + }) + + throw e + } + + bail(e) + } + }) + }, + }, + + /** + * Transaction state guards + */ + { + initProducerId: { legalStates: [STATES.UNINITIALIZED, STATES.READY] }, + beginTransaction: { legalStates: [STATES.READY], async: false }, + addPartitionsToTransaction: { legalStates: [STATES.TRANSACTING] }, + sendOffsets: { legalStates: [STATES.TRANSACTING] }, + commit: { legalStates: [STATES.TRANSACTING] }, + abort: { legalStates: [STATES.TRANSACTING] }, + } + ) + + return eosManager +} diff --git a/node_modules/kafkajs/src/producer/eosManager/transactionStateMachine.js b/node_modules/kafkajs/src/producer/eosManager/transactionStateMachine.js new file mode 100644 index 0000000..779b028 --- /dev/null +++ b/node_modules/kafkajs/src/producer/eosManager/transactionStateMachine.js @@ -0,0 +1,79 @@ +const { EventEmitter } = require('events') +const { KafkaJSNonRetriableError } = require('../../errors') +const STATES = require('./transactionStates') + +const VALID_STATE_TRANSITIONS = { + [STATES.UNINITIALIZED]: [STATES.READY], + [STATES.READY]: [STATES.READY, STATES.TRANSACTING], + [STATES.TRANSACTING]: [STATES.COMMITTING, STATES.ABORTING], + [STATES.COMMITTING]: [STATES.READY], + [STATES.ABORTING]: [STATES.READY], +} + +module.exports = ({ logger, initialState = STATES.UNINITIALIZED }) => { + let currentState = initialState + + const guard = (object, method, { legalStates, async: isAsync = true }) => { + if (!object[method]) { + throw new KafkaJSNonRetriableError(`Cannot add guard on missing method "${method}"`) + } + + return (...args) => { + const fn = object[method] + + if (!legalStates.includes(currentState)) { + const error = new KafkaJSNonRetriableError( + `Transaction state exception: Cannot call "${method}" in state "${currentState}"` + ) + + if (isAsync) { + return Promise.reject(error) + } else { + throw error + } + } + + return fn.apply(object, args) + } + } + + const stateMachine = Object.assign(new EventEmitter(), { + /** + * Create a clone of "object" where we ensure state machine is in correct state + * prior to calling any of the configured methods + * @param {Object} object The object whose methods we will guard + * @param {Object} methodStateMapping Keys are method names on "object" + * @param {string[]} methodStateMapping.legalStates Legal states for this method + * @param {boolean=true} methodStateMapping.async Whether this method is async (throw vs reject) + */ + createGuarded(object, methodStateMapping) { + const guardedMethods = Object.keys(methodStateMapping).reduce((guards, method) => { + guards[method] = guard(object, method, methodStateMapping[method]) + return guards + }, {}) + + return { ...object, ...guardedMethods } + }, + /** + * Transition safely to a new state + */ + transitionTo(state) { + logger.debug(`Transaction state transition ${currentState} --> ${state}`) + + if (!VALID_STATE_TRANSITIONS[currentState].includes(state)) { + throw new KafkaJSNonRetriableError( + `Transaction state exception: Invalid transition ${currentState} --> ${state}` + ) + } + + stateMachine.emit('transition', { to: state, from: currentState }) + currentState = state + }, + + state() { + return currentState + }, + }) + + return stateMachine +} diff --git a/node_modules/kafkajs/src/producer/eosManager/transactionStates.js b/node_modules/kafkajs/src/producer/eosManager/transactionStates.js new file mode 100644 index 0000000..6c2594d --- /dev/null +++ b/node_modules/kafkajs/src/producer/eosManager/transactionStates.js @@ -0,0 +1,7 @@ +module.exports = { + UNINITIALIZED: 'UNINITIALIZED', + READY: 'READY', + TRANSACTING: 'TRANSACTING', + COMMITTING: 'COMMITTING', + ABORTING: 'ABORTING', +} diff --git a/node_modules/kafkajs/src/producer/groupMessagesPerPartition.js b/node_modules/kafkajs/src/producer/groupMessagesPerPartition.js new file mode 100644 index 0000000..69b2909 --- /dev/null +++ b/node_modules/kafkajs/src/producer/groupMessagesPerPartition.js @@ -0,0 +1,11 @@ +module.exports = ({ topic, partitionMetadata, messages, partitioner }) => { + if (partitionMetadata.length === 0) { + return {} + } + + return messages.reduce((result, message) => { + const partition = partitioner({ topic, partitionMetadata, message }) + const current = result[partition] || [] + return Object.assign(result, { [partition]: [...current, message] }) + }, {}) +} diff --git a/node_modules/kafkajs/src/producer/index.js b/node_modules/kafkajs/src/producer/index.js new file mode 100644 index 0000000..785c649 --- /dev/null +++ b/node_modules/kafkajs/src/producer/index.js @@ -0,0 +1,246 @@ +const createRetry = require('../retry') +const { CONNECTION_STATUS } = require('../network/connectionStatus') +const { DefaultPartitioner } = require('./partitioners/') +const InstrumentationEventEmitter = require('../instrumentation/emitter') +const createEosManager = require('./eosManager') +const createMessageProducer = require('./messageProducer') +const { events, wrap: wrapEvent, unwrap: unwrapEvent } = require('./instrumentationEvents') +const { KafkaJSNonRetriableError } = require('../errors') + +const { values, keys } = Object +const eventNames = values(events) +const eventKeys = keys(events) + .map(key => `producer.events.${key}`) + .join(', ') + +const { CONNECT, DISCONNECT } = events + +/** + * + * @param {Object} params + * @param {import('../../types').Cluster} params.cluster + * @param {import('../../types').Logger} params.logger + * @param {import('../../types').ICustomPartitioner} [params.createPartitioner] + * @param {import('../../types').RetryOptions} [params.retry] + * @param {boolean} [params.idempotent] + * @param {string} [params.transactionalId] + * @param {number} [params.transactionTimeout] + * @param {InstrumentationEventEmitter} [params.instrumentationEmitter] + * + * @returns {import('../../types').Producer} + */ +module.exports = ({ + cluster, + logger: rootLogger, + createPartitioner = DefaultPartitioner, + retry, + idempotent = false, + transactionalId, + transactionTimeout, + instrumentationEmitter: rootInstrumentationEmitter, +}) => { + let connectionStatus = CONNECTION_STATUS.DISCONNECTED + retry = retry || { retries: idempotent ? Number.MAX_SAFE_INTEGER : 5 } + + if (idempotent && retry.retries < 1) { + throw new KafkaJSNonRetriableError( + 'Idempotent producer must allow retries to protect against transient errors' + ) + } + + const logger = rootLogger.namespace('Producer') + + if (idempotent && retry.retries < Number.MAX_SAFE_INTEGER) { + logger.warn('Limiting retries for the idempotent producer may invalidate EoS guarantees') + } + + const partitioner = createPartitioner() + const retrier = createRetry(Object.assign({}, cluster.retry, retry)) + const instrumentationEmitter = rootInstrumentationEmitter || new InstrumentationEventEmitter() + const idempotentEosManager = createEosManager({ + logger, + cluster, + transactionTimeout, + transactional: false, + transactionalId, + }) + + const { send, sendBatch } = createMessageProducer({ + logger, + cluster, + partitioner, + eosManager: idempotentEosManager, + idempotent, + retrier, + getConnectionStatus: () => connectionStatus, + }) + + let transactionalEosManager + + /** @type {import("../../types").Producer["on"]} */ + const on = (eventName, listener) => { + if (!eventNames.includes(eventName)) { + throw new KafkaJSNonRetriableError(`Event name should be one of ${eventKeys}`) + } + + return instrumentationEmitter.addListener(unwrapEvent(eventName), event => { + event.type = wrapEvent(event.type) + Promise.resolve(listener(event)).catch(e => { + logger.error(`Failed to execute listener: ${e.message}`, { + eventName, + stack: e.stack, + }) + }) + }) + } + + /** + * Begin a transaction. The returned object contains methods to send messages + * to the transaction and end the transaction by committing or aborting. + * + * Only messages sent on the transaction object will participate in the transaction. + * + * Calling any of the transactional methods after the transaction has ended + * will raise an exception (use `isActive` to ascertain if ended). + * @returns {Promise} + * + * @typedef {Object} Transaction + * @property {Function} send Identical to the producer "send" method + * @property {Function} sendBatch Identical to the producer "sendBatch" method + * @property {Function} abort Abort the transaction + * @property {Function} commit Commit the transaction + * @property {Function} isActive Whether the transaction is active + */ + const transaction = async () => { + if (!transactionalId) { + throw new KafkaJSNonRetriableError('Must provide transactional id for transactional producer') + } + + let transactionDidEnd = false + transactionalEosManager = + transactionalEosManager || + createEosManager({ + logger, + cluster, + transactionTimeout, + transactional: true, + transactionalId, + }) + + if (transactionalEosManager.isInTransaction()) { + throw new KafkaJSNonRetriableError( + 'There is already an ongoing transaction for this producer. Please end the transaction before beginning another.' + ) + } + + // We only initialize the producer id once + if (!transactionalEosManager.isInitialized()) { + await transactionalEosManager.initProducerId() + } + transactionalEosManager.beginTransaction() + + const { send: sendTxn, sendBatch: sendBatchTxn } = createMessageProducer({ + logger, + cluster, + partitioner, + retrier, + eosManager: transactionalEosManager, + idempotent: true, + getConnectionStatus: () => connectionStatus, + }) + + const isActive = () => transactionalEosManager.isInTransaction() && !transactionDidEnd + + const transactionGuard = fn => (...args) => { + if (!isActive()) { + return Promise.reject( + new KafkaJSNonRetriableError('Cannot continue to use transaction once ended') + ) + } + + return fn(...args) + } + + return { + sendBatch: transactionGuard(sendBatchTxn), + send: transactionGuard(sendTxn), + /** + * Abort the ongoing transaction. + * + * @throws {KafkaJSNonRetriableError} If transaction has ended + */ + abort: transactionGuard(async () => { + await transactionalEosManager.abort() + transactionDidEnd = true + }), + /** + * Commit the ongoing transaction. + * + * @throws {KafkaJSNonRetriableError} If transaction has ended + */ + commit: transactionGuard(async () => { + await transactionalEosManager.commit() + transactionDidEnd = true + }), + /** + * Sends a list of specified offsets to the consumer group coordinator, and also marks those offsets as part of the current transaction. + * + * @throws {KafkaJSNonRetriableError} If transaction has ended + */ + sendOffsets: transactionGuard(async ({ consumerGroupId, topics }) => { + await transactionalEosManager.sendOffsets({ consumerGroupId, topics }) + + for (const topicOffsets of topics) { + const { topic, partitions } = topicOffsets + for (const { partition, offset } of partitions) { + cluster.markOffsetAsCommitted({ + groupId: consumerGroupId, + topic, + partition, + offset, + }) + } + } + }), + isActive, + } + } + + /** + * @returns {Object} logger + */ + const getLogger = () => logger + + return { + /** + * @returns {Promise} + */ + connect: async () => { + await cluster.connect() + connectionStatus = CONNECTION_STATUS.CONNECTED + instrumentationEmitter.emit(CONNECT) + + if (idempotent && !idempotentEosManager.isInitialized()) { + await idempotentEosManager.initProducerId() + } + }, + /** + * @return {Promise} + */ + disconnect: async () => { + connectionStatus = CONNECTION_STATUS.DISCONNECTING + await cluster.disconnect() + connectionStatus = CONNECTION_STATUS.DISCONNECTED + instrumentationEmitter.emit(DISCONNECT) + }, + isIdempotent: () => { + return idempotent + }, + events, + on, + send, + sendBatch, + transaction, + logger: getLogger, + } +} diff --git a/node_modules/kafkajs/src/producer/instrumentationEvents.js b/node_modules/kafkajs/src/producer/instrumentationEvents.js new file mode 100644 index 0000000..ea46915 --- /dev/null +++ b/node_modules/kafkajs/src/producer/instrumentationEvents.js @@ -0,0 +1,28 @@ +const swapObject = require('../utils/swapObject') +const networkEvents = require('../network/instrumentationEvents') +const InstrumentationEventType = require('../instrumentation/eventType') +const producerType = InstrumentationEventType('producer') + +const events = { + CONNECT: producerType('connect'), + DISCONNECT: producerType('disconnect'), + REQUEST: producerType(networkEvents.NETWORK_REQUEST), + REQUEST_TIMEOUT: producerType(networkEvents.NETWORK_REQUEST_TIMEOUT), + REQUEST_QUEUE_SIZE: producerType(networkEvents.NETWORK_REQUEST_QUEUE_SIZE), +} + +const wrappedEvents = { + [events.REQUEST]: networkEvents.NETWORK_REQUEST, + [events.REQUEST_TIMEOUT]: networkEvents.NETWORK_REQUEST_TIMEOUT, + [events.REQUEST_QUEUE_SIZE]: networkEvents.NETWORK_REQUEST_QUEUE_SIZE, +} + +const reversedWrappedEvents = swapObject(wrappedEvents) +const unwrap = eventName => wrappedEvents[eventName] || eventName +const wrap = eventName => reversedWrappedEvents[eventName] || eventName + +module.exports = { + events, + wrap, + unwrap, +} diff --git a/node_modules/kafkajs/src/producer/messageProducer.js b/node_modules/kafkajs/src/producer/messageProducer.js new file mode 100644 index 0000000..ef45c8a --- /dev/null +++ b/node_modules/kafkajs/src/producer/messageProducer.js @@ -0,0 +1,132 @@ +const createSendMessages = require('./sendMessages') +const { KafkaJSError, KafkaJSNonRetriableError } = require('../errors') +const { CONNECTION_STATUS } = require('../network/connectionStatus') + +module.exports = ({ + logger, + cluster, + partitioner, + eosManager, + idempotent, + retrier, + getConnectionStatus, +}) => { + const sendMessages = createSendMessages({ + logger, + cluster, + retrier, + partitioner, + eosManager, + }) + + const validateConnectionStatus = () => { + const connectionStatus = getConnectionStatus() + + switch (connectionStatus) { + case CONNECTION_STATUS.DISCONNECTING: + throw new KafkaJSNonRetriableError( + `The producer is disconnecting; therefore, it can't safely accept messages anymore` + ) + case CONNECTION_STATUS.DISCONNECTED: + throw new KafkaJSError('The producer is disconnected') + } + } + + /** + * @typedef {Object} TopicMessages + * @property {string} topic + * @property {Array} messages An array of objects with "key" and "value", example: + * [{ key: 'my-key', value: 'my-value'}] + * + * @typedef {Object} SendBatchRequest + * @property {Array} topicMessages + * @property {number} [acks=-1] Control the number of required acks. + * -1 = all replicas must acknowledge + * 0 = no acknowledgments + * 1 = only waits for the leader to acknowledge + * + * @property {number} [timeout=30000] The time to await a response in ms + * @property {Compression.Types} [compression=Compression.Types.None] Compression codec + * + * @param {SendBatchRequest} + * @returns {Promise} + */ + const sendBatch = async ({ acks = -1, timeout, compression, topicMessages = [] }) => { + if (topicMessages.some(({ topic }) => !topic)) { + throw new KafkaJSNonRetriableError(`Invalid topic`) + } + + if (idempotent && acks !== -1) { + throw new KafkaJSNonRetriableError( + `Not requiring ack for all messages invalidates the idempotent producer's EoS guarantees` + ) + } + + for (const { topic, messages } of topicMessages) { + if (!messages) { + throw new KafkaJSNonRetriableError( + `Invalid messages array [${messages}] for topic "${topic}"` + ) + } + + const messageWithoutValue = messages.find(message => message.value === undefined) + if (messageWithoutValue) { + throw new KafkaJSNonRetriableError( + `Invalid message without value for topic "${topic}": ${JSON.stringify( + messageWithoutValue + )}` + ) + } + } + + validateConnectionStatus() + const mergedTopicMessages = topicMessages.reduce((merged, { topic, messages }) => { + const index = merged.findIndex(({ topic: mergedTopic }) => topic === mergedTopic) + + if (index === -1) { + merged.push({ topic, messages }) + } else { + merged[index].messages = [...merged[index].messages, ...messages] + } + + return merged + }, []) + + return await sendMessages({ + acks, + timeout, + compression, + topicMessages: mergedTopicMessages, + }) + } + + /** + * @param {ProduceRequest} ProduceRequest + * @returns {Promise} + * + * @typedef {Object} ProduceRequest + * @property {string} topic + * @property {Array} messages An array of objects with "key" and "value", example: + * [{ key: 'my-key', value: 'my-value'}] + * @property {number} [acks=-1] Control the number of required acks. + * -1 = all replicas must acknowledge + * 0 = no acknowledgments + * 1 = only waits for the leader to acknowledge + * @property {number} [timeout=30000] The time to await a response in ms + * @property {Compression.Types} [compression=Compression.Types.None] Compression codec + */ + const send = async ({ acks, timeout, compression, topic, messages }) => { + const topicMessage = { topic, messages } + return sendBatch({ + acks, + timeout, + compression, + topicMessages: [topicMessage], + }) + } + + return { + send, + sendBatch, + } +} diff --git a/node_modules/kafkajs/src/producer/partitioners/default/index.js b/node_modules/kafkajs/src/producer/partitioners/default/index.js new file mode 100644 index 0000000..cbdc4f0 --- /dev/null +++ b/node_modules/kafkajs/src/producer/partitioners/default/index.js @@ -0,0 +1,4 @@ +const murmur2 = require('./murmur2') +const createDefaultPartitioner = require('../legacy/partitioner') + +module.exports = createDefaultPartitioner(murmur2) diff --git a/node_modules/kafkajs/src/producer/partitioners/default/murmur2.js b/node_modules/kafkajs/src/producer/partitioners/default/murmur2.js new file mode 100644 index 0000000..02e531d --- /dev/null +++ b/node_modules/kafkajs/src/producer/partitioners/default/murmur2.js @@ -0,0 +1,53 @@ +/* eslint-disable */ +const Long = require('../../../utils/long') + +// Based on the kafka client 0.10.2 murmur2 implementation +// https://github.com/apache/kafka/blob/0.10.2/clients/src/main/java/org/apache/kafka/common/utils/Utils.java#L364 + +const SEED = Long.fromValue(0x9747b28c) + +// 'm' and 'r' are mixing constants generated offline. +// They're not really 'magic', they just happen to work well. +const M = Long.fromValue(0x5bd1e995) +const R = Long.fromValue(24) + +module.exports = key => { + const data = Buffer.isBuffer(key) ? key : Buffer.from(String(key)) + const length = data.length + + // Initialize the hash to a random value + let h = Long.fromValue(SEED.xor(length)) + let length4 = Math.floor(length / 4) + + for (let i = 0; i < length4; i++) { + const i4 = i * 4 + let k = + (data[i4 + 0] & 0xff) + + ((data[i4 + 1] & 0xff) << 8) + + ((data[i4 + 2] & 0xff) << 16) + + ((data[i4 + 3] & 0xff) << 24) + k = Long.fromValue(k) + k = k.multiply(M) + k = k.xor(k.toInt() >>> R) + k = Long.fromValue(k).multiply(M) + h = h.multiply(M) + h = h.xor(k) + } + + // Handle the last few bytes of the input array + switch (length % 4) { + case 3: + h = h.xor((data[(length & ~3) + 2] & 0xff) << 16) + case 2: + h = h.xor((data[(length & ~3) + 1] & 0xff) << 8) + case 1: + h = h.xor(data[length & ~3] & 0xff) + h = h.multiply(M) + } + + h = h.xor(h.toInt() >>> 13) + h = h.multiply(M) + h = h.xor(h.toInt() >>> 15) + + return h.toInt() +} diff --git a/node_modules/kafkajs/src/producer/partitioners/index.js b/node_modules/kafkajs/src/producer/partitioners/index.js new file mode 100644 index 0000000..9fbad60 --- /dev/null +++ b/node_modules/kafkajs/src/producer/partitioners/index.js @@ -0,0 +1,14 @@ +const DefaultPartitioner = require('./default') +const LegacyPartitioner = require('./legacy') + +module.exports = { + DefaultPartitioner, + LegacyPartitioner, + /** + * @deprecated Use DefaultPartitioner instead + * + * The JavaCompatiblePartitioner was renamed DefaultPartitioner + * and made to be the default in 2.0.0. + */ + JavaCompatiblePartitioner: DefaultPartitioner, +} diff --git a/node_modules/kafkajs/src/producer/partitioners/legacy/index.js b/node_modules/kafkajs/src/producer/partitioners/legacy/index.js new file mode 100644 index 0000000..be3ce76 --- /dev/null +++ b/node_modules/kafkajs/src/producer/partitioners/legacy/index.js @@ -0,0 +1,4 @@ +const murmur2 = require('./murmur2') +const createLegacyPartitioner = require('./partitioner') + +module.exports = createLegacyPartitioner(murmur2) diff --git a/node_modules/kafkajs/src/producer/partitioners/legacy/murmur2.js b/node_modules/kafkajs/src/producer/partitioners/legacy/murmur2.js new file mode 100644 index 0000000..3b5661e --- /dev/null +++ b/node_modules/kafkajs/src/producer/partitioners/legacy/murmur2.js @@ -0,0 +1,51 @@ +/* eslint-disable */ + +// Based on the kafka client 0.10.2 murmur2 implementation +// https://github.com/apache/kafka/blob/0.10.2/clients/src/main/java/org/apache/kafka/common/utils/Utils.java#L364 + +const SEED = 0x9747b28c + +// 'm' and 'r' are mixing constants generated offline. +// They're not really 'magic', they just happen to work well. +const M = 0x5bd1e995 +const R = 24 + +module.exports = key => { + const data = Buffer.isBuffer(key) ? key : Buffer.from(String(key)) + const length = data.length + + // Initialize the hash to a random value + let h = SEED ^ length + let length4 = length / 4 + + for (let i = 0; i < length4; i++) { + const i4 = i * 4 + let k = + (data[i4 + 0] & 0xff) + + ((data[i4 + 1] & 0xff) << 8) + + ((data[i4 + 2] & 0xff) << 16) + + ((data[i4 + 3] & 0xff) << 24) + k *= M + k ^= k >>> R + k *= M + h *= M + h ^= k + } + + // Handle the last few bytes of the input array + switch (length % 4) { + case 3: + h ^= (data[(length & ~3) + 2] & 0xff) << 16 + case 2: + h ^= (data[(length & ~3) + 1] & 0xff) << 8 + case 1: + h ^= data[length & ~3] & 0xff + h *= M + } + + h ^= h >>> 13 + h *= M + h ^= h >>> 15 + + return h +} diff --git a/node_modules/kafkajs/src/producer/partitioners/legacy/partitioner.js b/node_modules/kafkajs/src/producer/partitioners/legacy/partitioner.js new file mode 100644 index 0000000..413d97e --- /dev/null +++ b/node_modules/kafkajs/src/producer/partitioners/legacy/partitioner.js @@ -0,0 +1,47 @@ +const randomBytes = require('./randomBytes') + +// Based on the java client 0.10.2 +// https://github.com/apache/kafka/blob/0.10.2/clients/src/main/java/org/apache/kafka/clients/producer/internals/DefaultPartitioner.java + +/** + * A cheap way to deterministically convert a number to a positive value. When the input is + * positive, the original value is returned. When the input number is negative, the returned + * positive value is the original value bit AND against 0x7fffffff which is not its absolutely + * value. + */ +const toPositive = x => x & 0x7fffffff + +/** + * The default partitioning strategy: + * - If a partition is specified in the message, use it + * - If no partition is specified but a key is present choose a partition based on a hash of the key + * - If no partition or key is present choose a partition in a round-robin fashion + */ +module.exports = murmur2 => () => { + const counters = {} + + return ({ topic, partitionMetadata, message }) => { + if (!(topic in counters)) { + counters[topic] = randomBytes(32).readUInt32BE(0) + } + const numPartitions = partitionMetadata.length + const availablePartitions = partitionMetadata.filter(p => p.leader >= 0) + const numAvailablePartitions = availablePartitions.length + + if (message.partition !== null && message.partition !== undefined) { + return message.partition + } + + if (message.key !== null && message.key !== undefined) { + return toPositive(murmur2(message.key)) % numPartitions + } + + if (numAvailablePartitions > 0) { + const i = toPositive(++counters[topic]) % numAvailablePartitions + return availablePartitions[i].partitionId + } + + // no partitions are available, give a non-available partition + return toPositive(++counters[topic]) % numPartitions + } +} diff --git a/node_modules/kafkajs/src/producer/partitioners/legacy/randomBytes.js b/node_modules/kafkajs/src/producer/partitioners/legacy/randomBytes.js new file mode 100644 index 0000000..a229f3c --- /dev/null +++ b/node_modules/kafkajs/src/producer/partitioners/legacy/randomBytes.js @@ -0,0 +1,31 @@ +const { KafkaJSNonRetriableError } = require('../../../errors') + +const toNodeCompatible = crypto => ({ + randomBytes: size => crypto.getRandomValues(Buffer.allocUnsafe(size)), +}) + +let cryptoImplementation = null +if (global && global.crypto) { + cryptoImplementation = + global.crypto.randomBytes === undefined ? toNodeCompatible(global.crypto) : global.crypto +} else if (global && global.msCrypto) { + cryptoImplementation = toNodeCompatible(global.msCrypto) +} else if (global && !global.crypto) { + cryptoImplementation = require('crypto') +} + +const MAX_BYTES = 65536 + +module.exports = size => { + if (size > MAX_BYTES) { + throw new KafkaJSNonRetriableError( + `Byte length (${size}) exceeds the max number of bytes of entropy available (${MAX_BYTES})` + ) + } + + if (!cryptoImplementation) { + throw new KafkaJSNonRetriableError('No available crypto implementation') + } + + return cryptoImplementation.randomBytes(size) +} diff --git a/node_modules/kafkajs/src/producer/responseSerializer.js b/node_modules/kafkajs/src/producer/responseSerializer.js new file mode 100644 index 0000000..7db9b57 --- /dev/null +++ b/node_modules/kafkajs/src/producer/responseSerializer.js @@ -0,0 +1,4 @@ +module.exports = ({ topics }) => + topics.flatMap(({ topicName, partitions }) => + partitions.map(partition => ({ topicName, ...partition })) + ) diff --git a/node_modules/kafkajs/src/producer/sendMessages.js b/node_modules/kafkajs/src/producer/sendMessages.js new file mode 100644 index 0000000..bc675e3 --- /dev/null +++ b/node_modules/kafkajs/src/producer/sendMessages.js @@ -0,0 +1,170 @@ +const { KafkaJSMetadataNotLoaded } = require('../errors') +const { staleMetadata } = require('../protocol/error') +const groupMessagesPerPartition = require('./groupMessagesPerPartition') +const createTopicData = require('./createTopicData') +const responseSerializer = require('./responseSerializer') + +const { keys } = Object + +/** + * @param {Object} options + * @param {import("../../types").Logger} options.logger + * @param {import("../../types").Cluster} options.cluster + * @param {ReturnType} options.partitioner + * @param {import("./eosManager").EosManager} options.eosManager + * @param {import("../retry").Retrier} options.retrier + */ +module.exports = ({ logger, cluster, partitioner, eosManager, retrier }) => { + return async ({ acks, timeout, compression, topicMessages }) => { + /** @type {Map} */ + const responsePerBroker = new Map() + + /** @param {Map} responsePerBroker */ + const createProducerRequests = async responsePerBroker => { + const topicMetadata = new Map() + + await cluster.refreshMetadataIfNecessary() + + for (const { topic, messages } of topicMessages) { + const partitionMetadata = cluster.findTopicPartitionMetadata(topic) + + if (partitionMetadata.length === 0) { + logger.debug('Producing to topic without metadata', { + topic, + targetTopics: Array.from(cluster.targetTopics), + }) + + throw new KafkaJSMetadataNotLoaded('Producing to topic without metadata') + } + + const messagesPerPartition = groupMessagesPerPartition({ + topic, + partitionMetadata, + messages, + partitioner, + }) + + const partitions = keys(messagesPerPartition) + const partitionsPerLeader = cluster.findLeaderForPartitions(topic, partitions) + const leaders = keys(partitionsPerLeader) + + topicMetadata.set(topic, { + partitionsPerLeader, + messagesPerPartition, + }) + + for (const nodeId of leaders) { + const broker = await cluster.findBroker({ nodeId }) + if (!responsePerBroker.has(broker)) { + responsePerBroker.set(broker, null) + } + } + } + + const brokers = Array.from(responsePerBroker.keys()) + const brokersWithoutResponse = brokers.filter(broker => !responsePerBroker.get(broker)) + + return brokersWithoutResponse.map(async broker => { + const entries = Array.from(topicMetadata.entries()) + const topicDataForBroker = entries + .filter(([_, { partitionsPerLeader }]) => !!partitionsPerLeader[broker.nodeId]) + .map(([topic, { partitionsPerLeader, messagesPerPartition, sequencePerPartition }]) => ({ + topic, + partitions: partitionsPerLeader[broker.nodeId], + messagesPerPartition, + })) + + const topicData = createTopicData(topicDataForBroker) + + await eosManager.acquireBrokerLock(broker) + try { + if (eosManager.isTransactional()) { + await eosManager.addPartitionsToTransaction(topicData) + } + + topicData.forEach(({ topic, partitions }) => { + partitions.forEach(entry => { + entry['firstSequence'] = eosManager.getSequence(topic, entry.partition) + eosManager.updateSequence(topic, entry.partition, entry.messages.length) + }) + }) + + let response + try { + response = await broker.produce({ + transactionalId: eosManager.isTransactional() + ? eosManager.getTransactionalId() + : undefined, + producerId: eosManager.getProducerId(), + producerEpoch: eosManager.getProducerEpoch(), + acks, + timeout, + compression, + topicData, + }) + } catch (e) { + topicData.forEach(({ topic, partitions }) => { + partitions.forEach(entry => { + eosManager.updateSequence(topic, entry.partition, -entry.messages.length) + }) + }) + throw e + } + + const expectResponse = acks !== 0 + const formattedResponse = expectResponse ? responseSerializer(response) : [] + + responsePerBroker.set(broker, formattedResponse) + } catch (e) { + responsePerBroker.delete(broker) + throw e + } finally { + await eosManager.releaseBrokerLock(broker) + } + }) + } + + return retrier(async (bail, retryCount, retryTime) => { + const topics = topicMessages.map(({ topic }) => topic) + await cluster.addMultipleTargetTopics(topics) + + try { + const requests = await createProducerRequests(responsePerBroker) + await Promise.all(requests) + return Array.from(responsePerBroker.values()).flat() + } catch (e) { + if (e.name === 'KafkaJSConnectionClosedError') { + cluster.removeBroker({ host: e.host, port: e.port }) + } + + if (!cluster.isConnected()) { + logger.debug(`Cluster has disconnected, reconnecting: ${e.message}`, { + retryCount, + retryTime, + }) + await cluster.connect() + await cluster.refreshMetadata() + throw e + } + + // This is necessary in case the metadata is stale and the number of partitions + // for this topic has increased in the meantime + if ( + staleMetadata(e) || + e.name === 'KafkaJSMetadataNotLoaded' || + e.name === 'KafkaJSConnectionError' || + e.name === 'KafkaJSConnectionClosedError' || + (e.name === 'KafkaJSProtocolError' && e.retriable) + ) { + logger.error(`Failed to send messages: ${e.message}`, { retryCount, retryTime }) + await cluster.refreshMetadata() + throw e + } + + logger.error(`${e.message}`, { retryCount, retryTime }) + if (e.retriable) throw e + bail(e) + } + }) + } +} diff --git a/node_modules/kafkajs/src/protocol/aclOperationTypes.js b/node_modules/kafkajs/src/protocol/aclOperationTypes.js new file mode 100644 index 0000000..81e844c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/aclOperationTypes.js @@ -0,0 +1,65 @@ +// From: +// https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclOperation.java#L44 + +/** + * @typedef {number} ACLOperationTypes + * + * Enum for ACL Operations Types + * @readonly + * @enum {ACLOperationTypes} + */ +module.exports = { + /** + * Represents any AclOperation which this client cannot understand, perhaps because this + * client is too old. + */ + UNKNOWN: 0, + /** + * In a filter, matches any AclOperation. + */ + ANY: 1, + /** + * ALL operation. + */ + ALL: 2, + /** + * READ operation. + */ + READ: 3, + /** + * WRITE operation. + */ + WRITE: 4, + /** + * CREATE operation. + */ + CREATE: 5, + /** + * DELETE operation. + */ + DELETE: 6, + /** + * ALTER operation. + */ + ALTER: 7, + /** + * DESCRIBE operation. + */ + DESCRIBE: 8, + /** + * CLUSTER_ACTION operation. + */ + CLUSTER_ACTION: 9, + /** + * DESCRIBE_CONFIGS operation. + */ + DESCRIBE_CONFIGS: 10, + /** + * ALTER_CONFIGS operation. + */ + ALTER_CONFIGS: 11, + /** + * IDEMPOTENT_WRITE operation. + */ + IDEMPOTENT_WRITE: 12, +} diff --git a/node_modules/kafkajs/src/protocol/aclPermissionTypes.js b/node_modules/kafkajs/src/protocol/aclPermissionTypes.js new file mode 100644 index 0000000..bd91310 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/aclPermissionTypes.js @@ -0,0 +1,29 @@ +// From: +// https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclPermissionType.java/#L31 + +/** + * @typedef {number} ACLPermissionTypes + * + * Enum for Permission Types + * @readonly + * @enum {ACLPermissionTypes} + */ +module.exports = { + /** + * Represents any AclPermissionType which this client cannot understand, + * perhaps because this client is too old. + */ + UNKNOWN: 0, + /** + * In a filter, matches any AclPermissionType. + */ + ANY: 1, + /** + * Disallows access. + */ + DENY: 2, + /** + * Grants access. + */ + ALLOW: 3, +} diff --git a/node_modules/kafkajs/src/protocol/aclResourceTypes.js b/node_modules/kafkajs/src/protocol/aclResourceTypes.js new file mode 100644 index 0000000..48c4e02 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/aclResourceTypes.js @@ -0,0 +1,42 @@ +/** + * @see https://github.com/apache/kafka/blob/a15387f34d142684859c2a57fcbef25edcdce25a/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java#L25-L31 + * @typedef {number} ACLResourceTypes + * + * Enum for ACL Resource Types + * @readonly + * @enum {ACLResourceTypes} + */ + +module.exports = { + /** + * Represents any ResourceType which this client cannot understand, + * perhaps because this client is too old. + */ + UNKNOWN: 0, + /** + * In a filter, matches any ResourceType. + */ + ANY: 1, + /** + * A Kafka topic. + * @see http://kafka.apache.org/documentation/#topicconfigs + */ + TOPIC: 2, + /** + * A consumer group. + * @see http://kafka.apache.org/documentation/#consumerconfigs + */ + GROUP: 3, + /** + * The cluster as a whole. + */ + CLUSTER: 4, + /** + * A transactional ID. + */ + TRANSACTIONAL_ID: 5, + /** + * A token ID. + */ + DELEGATION_TOKEN: 6, +} diff --git a/node_modules/kafkajs/src/protocol/configResourceTypes.js b/node_modules/kafkajs/src/protocol/configResourceTypes.js new file mode 100644 index 0000000..e74e91c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/configResourceTypes.js @@ -0,0 +1,9 @@ +/** + * @see https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java + */ +module.exports = { + UNKNOWN: 0, + TOPIC: 2, + BROKER: 4, + BROKER_LOGGER: 8, +} diff --git a/node_modules/kafkajs/src/protocol/configSource.js b/node_modules/kafkajs/src/protocol/configSource.js new file mode 100644 index 0000000..98f5e05 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/configSource.js @@ -0,0 +1,12 @@ +/** + * @see https://github.com/apache/kafka/blob/1f240ce1793cab09e1c4823e17436d2b030df2bc/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java#L115-L122 + */ +module.exports = { + UNKNOWN: 0, + TOPIC_CONFIG: 1, + DYNAMIC_BROKER_CONFIG: 2, + DYNAMIC_DEFAULT_BROKER_CONFIG: 3, + STATIC_BROKER_CONFIG: 4, + DEFAULT_CONFIG: 5, + DYNAMIC_BROKER_LOGGER_CONFIG: 6, +} diff --git a/node_modules/kafkajs/src/protocol/coordinatorTypes.js b/node_modules/kafkajs/src/protocol/coordinatorTypes.js new file mode 100644 index 0000000..62eda6c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/coordinatorTypes.js @@ -0,0 +1,12 @@ +// From: https://kafka.apache.org/protocol.html#The_Messages_FindCoordinator + +/** + * @typedef {number} CoordinatorType + * + * Enum for the types of coordinator to find. + * @enum {CoordinatorType} + */ +module.exports = { + GROUP: 0, + TRANSACTION: 1, +} diff --git a/node_modules/kafkajs/src/protocol/crc32.js b/node_modules/kafkajs/src/protocol/crc32.js new file mode 100644 index 0000000..6dd48b3 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/crc32.js @@ -0,0 +1,270 @@ +// Based on https://github.com/brianloveswords/buffer-crc32/blob/master/index.js + +var CRC_TABLE = new Int32Array([ + 0x00000000, + 0x77073096, + 0xee0e612c, + 0x990951ba, + 0x076dc419, + 0x706af48f, + 0xe963a535, + 0x9e6495a3, + 0x0edb8832, + 0x79dcb8a4, + 0xe0d5e91e, + 0x97d2d988, + 0x09b64c2b, + 0x7eb17cbd, + 0xe7b82d07, + 0x90bf1d91, + 0x1db71064, + 0x6ab020f2, + 0xf3b97148, + 0x84be41de, + 0x1adad47d, + 0x6ddde4eb, + 0xf4d4b551, + 0x83d385c7, + 0x136c9856, + 0x646ba8c0, + 0xfd62f97a, + 0x8a65c9ec, + 0x14015c4f, + 0x63066cd9, + 0xfa0f3d63, + 0x8d080df5, + 0x3b6e20c8, + 0x4c69105e, + 0xd56041e4, + 0xa2677172, + 0x3c03e4d1, + 0x4b04d447, + 0xd20d85fd, + 0xa50ab56b, + 0x35b5a8fa, + 0x42b2986c, + 0xdbbbc9d6, + 0xacbcf940, + 0x32d86ce3, + 0x45df5c75, + 0xdcd60dcf, + 0xabd13d59, + 0x26d930ac, + 0x51de003a, + 0xc8d75180, + 0xbfd06116, + 0x21b4f4b5, + 0x56b3c423, + 0xcfba9599, + 0xb8bda50f, + 0x2802b89e, + 0x5f058808, + 0xc60cd9b2, + 0xb10be924, + 0x2f6f7c87, + 0x58684c11, + 0xc1611dab, + 0xb6662d3d, + 0x76dc4190, + 0x01db7106, + 0x98d220bc, + 0xefd5102a, + 0x71b18589, + 0x06b6b51f, + 0x9fbfe4a5, + 0xe8b8d433, + 0x7807c9a2, + 0x0f00f934, + 0x9609a88e, + 0xe10e9818, + 0x7f6a0dbb, + 0x086d3d2d, + 0x91646c97, + 0xe6635c01, + 0x6b6b51f4, + 0x1c6c6162, + 0x856530d8, + 0xf262004e, + 0x6c0695ed, + 0x1b01a57b, + 0x8208f4c1, + 0xf50fc457, + 0x65b0d9c6, + 0x12b7e950, + 0x8bbeb8ea, + 0xfcb9887c, + 0x62dd1ddf, + 0x15da2d49, + 0x8cd37cf3, + 0xfbd44c65, + 0x4db26158, + 0x3ab551ce, + 0xa3bc0074, + 0xd4bb30e2, + 0x4adfa541, + 0x3dd895d7, + 0xa4d1c46d, + 0xd3d6f4fb, + 0x4369e96a, + 0x346ed9fc, + 0xad678846, + 0xda60b8d0, + 0x44042d73, + 0x33031de5, + 0xaa0a4c5f, + 0xdd0d7cc9, + 0x5005713c, + 0x270241aa, + 0xbe0b1010, + 0xc90c2086, + 0x5768b525, + 0x206f85b3, + 0xb966d409, + 0xce61e49f, + 0x5edef90e, + 0x29d9c998, + 0xb0d09822, + 0xc7d7a8b4, + 0x59b33d17, + 0x2eb40d81, + 0xb7bd5c3b, + 0xc0ba6cad, + 0xedb88320, + 0x9abfb3b6, + 0x03b6e20c, + 0x74b1d29a, + 0xead54739, + 0x9dd277af, + 0x04db2615, + 0x73dc1683, + 0xe3630b12, + 0x94643b84, + 0x0d6d6a3e, + 0x7a6a5aa8, + 0xe40ecf0b, + 0x9309ff9d, + 0x0a00ae27, + 0x7d079eb1, + 0xf00f9344, + 0x8708a3d2, + 0x1e01f268, + 0x6906c2fe, + 0xf762575d, + 0x806567cb, + 0x196c3671, + 0x6e6b06e7, + 0xfed41b76, + 0x89d32be0, + 0x10da7a5a, + 0x67dd4acc, + 0xf9b9df6f, + 0x8ebeeff9, + 0x17b7be43, + 0x60b08ed5, + 0xd6d6a3e8, + 0xa1d1937e, + 0x38d8c2c4, + 0x4fdff252, + 0xd1bb67f1, + 0xa6bc5767, + 0x3fb506dd, + 0x48b2364b, + 0xd80d2bda, + 0xaf0a1b4c, + 0x36034af6, + 0x41047a60, + 0xdf60efc3, + 0xa867df55, + 0x316e8eef, + 0x4669be79, + 0xcb61b38c, + 0xbc66831a, + 0x256fd2a0, + 0x5268e236, + 0xcc0c7795, + 0xbb0b4703, + 0x220216b9, + 0x5505262f, + 0xc5ba3bbe, + 0xb2bd0b28, + 0x2bb45a92, + 0x5cb36a04, + 0xc2d7ffa7, + 0xb5d0cf31, + 0x2cd99e8b, + 0x5bdeae1d, + 0x9b64c2b0, + 0xec63f226, + 0x756aa39c, + 0x026d930a, + 0x9c0906a9, + 0xeb0e363f, + 0x72076785, + 0x05005713, + 0x95bf4a82, + 0xe2b87a14, + 0x7bb12bae, + 0x0cb61b38, + 0x92d28e9b, + 0xe5d5be0d, + 0x7cdcefb7, + 0x0bdbdf21, + 0x86d3d2d4, + 0xf1d4e242, + 0x68ddb3f8, + 0x1fda836e, + 0x81be16cd, + 0xf6b9265b, + 0x6fb077e1, + 0x18b74777, + 0x88085ae6, + 0xff0f6a70, + 0x66063bca, + 0x11010b5c, + 0x8f659eff, + 0xf862ae69, + 0x616bffd3, + 0x166ccf45, + 0xa00ae278, + 0xd70dd2ee, + 0x4e048354, + 0x3903b3c2, + 0xa7672661, + 0xd06016f7, + 0x4969474d, + 0x3e6e77db, + 0xaed16a4a, + 0xd9d65adc, + 0x40df0b66, + 0x37d83bf0, + 0xa9bcae53, + 0xdebb9ec5, + 0x47b2cf7f, + 0x30b5ffe9, + 0xbdbdf21c, + 0xcabac28a, + 0x53b39330, + 0x24b4a3a6, + 0xbad03605, + 0xcdd70693, + 0x54de5729, + 0x23d967bf, + 0xb3667a2e, + 0xc4614ab8, + 0x5d681b02, + 0x2a6f2b94, + 0xb40bbe37, + 0xc30c8ea1, + 0x5a05df1b, + 0x2d02ef8d, +]) + +module.exports = encoder => { + const { buffer } = encoder + const l = buffer.length + let crc = -1 + for (let n = 0; n < l; n++) { + crc = CRC_TABLE[(crc ^ buffer[n]) & 0xff] ^ (crc >>> 8) + } + return crc ^ -1 +} diff --git a/node_modules/kafkajs/src/protocol/decoder.js b/node_modules/kafkajs/src/protocol/decoder.js new file mode 100644 index 0000000..e2834d0 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/decoder.js @@ -0,0 +1,309 @@ +const { KafkaJSInvalidVarIntError, KafkaJSInvalidLongError } = require('../errors') +const Long = require('../utils/long') + +const INT8_SIZE = 1 +const INT16_SIZE = 2 +const INT32_SIZE = 4 +const INT64_SIZE = 8 +const DOUBLE_SIZE = 8 + +const MOST_SIGNIFICANT_BIT = 0x80 // 128 +const OTHER_BITS = 0x7f // 127 + +module.exports = class Decoder { + static int32Size() { + return INT32_SIZE + } + + static decodeZigZag(value) { + return (value >>> 1) ^ -(value & 1) + } + + static decodeZigZag64(longValue) { + return longValue.shiftRightUnsigned(1).xor(longValue.and(Long.fromInt(1)).negate()) + } + + constructor(buffer) { + this.buffer = buffer + this.offset = 0 + } + + readInt8() { + const value = this.buffer.readInt8(this.offset) + this.offset += INT8_SIZE + return value + } + + canReadInt16() { + return this.canReadBytes(INT16_SIZE) + } + + readInt16() { + const value = this.buffer.readInt16BE(this.offset) + this.offset += INT16_SIZE + return value + } + + canReadInt32() { + return this.canReadBytes(INT32_SIZE) + } + + readInt32() { + const value = this.buffer.readInt32BE(this.offset) + this.offset += INT32_SIZE + return value + } + + canReadInt64() { + return this.canReadBytes(INT64_SIZE) + } + + readInt64() { + const first = this.buffer[this.offset] + const last = this.buffer[this.offset + 7] + + const low = + (first << 24) + // Overflow + this.buffer[this.offset + 1] * 2 ** 16 + + this.buffer[this.offset + 2] * 2 ** 8 + + this.buffer[this.offset + 3] + const high = + this.buffer[this.offset + 4] * 2 ** 24 + + this.buffer[this.offset + 5] * 2 ** 16 + + this.buffer[this.offset + 6] * 2 ** 8 + + last + this.offset += INT64_SIZE + + return (BigInt(low) << 32n) + BigInt(high) + } + + readDouble() { + const value = this.buffer.readDoubleBE(this.offset) + this.offset += DOUBLE_SIZE + return value + } + + readString() { + const byteLength = this.readInt16() + + if (byteLength === -1) { + return null + } + + const stringBuffer = this.buffer.slice(this.offset, this.offset + byteLength) + const value = stringBuffer.toString('utf8') + this.offset += byteLength + return value + } + + readVarIntString() { + const byteLength = this.readVarInt() + + if (byteLength === -1) { + return null + } + + const stringBuffer = this.buffer.slice(this.offset, this.offset + byteLength) + const value = stringBuffer.toString('utf8') + this.offset += byteLength + return value + } + + readUVarIntString() { + const byteLength = this.readUVarInt() + + if (byteLength === 0) { + return null + } + + const stringBuffer = this.buffer.slice(this.offset, this.offset + byteLength - 1) + const value = stringBuffer.toString('utf8') + + this.offset += byteLength - 1 + return value + } + + canReadBytes(length) { + return Buffer.byteLength(this.buffer) - this.offset >= length + } + + readBytes(byteLength = this.readInt32()) { + if (byteLength === -1) { + return null + } + + const stringBuffer = this.buffer.slice(this.offset, this.offset + byteLength) + this.offset += byteLength + return stringBuffer + } + + readVarIntBytes() { + const byteLength = this.readVarInt() + + if (byteLength === -1) { + return null + } + + const stringBuffer = this.buffer.slice(this.offset, this.offset + byteLength) + this.offset += byteLength + return stringBuffer + } + + readUVarIntBytes() { + const byteLength = this.readUVarInt() + + if (byteLength === 0) { + return null + } + + const stringBuffer = this.buffer.slice(this.offset, this.offset + byteLength) + this.offset += byteLength - 1 + return stringBuffer + } + + readBoolean() { + return this.readInt8() === 1 + } + + readAll() { + const result = this.buffer.slice(this.offset) + this.offset += Buffer.byteLength(this.buffer) + return result + } + + readArray(reader) { + const length = this.readInt32() + + if (length === -1) { + return [] + } + + const array = new Array(length) + for (let i = 0; i < length; i++) { + array[i] = reader(this) + } + + return array + } + + readVarIntArray(reader) { + const length = this.readVarInt() + + if (length === -1) { + return [] + } + + const array = new Array(length) + for (let i = 0; i < length; i++) { + array[i] = reader(this) + } + + return array + } + + /* According to the protocol type documentation: https://kafka.apache.org/protocol#protocol_types, + a compact array with length zero is a null array. An array with length 1 is an empty array. */ + readUVarIntArray(reader) { + const length = this.readUVarInt() + + if (length === 0) { + return null + } + + const array = new Array(length - 1) + for (let i = 0; i < length - 1; i++) { + array[i] = reader(this) + } + + return array + } + + async readArrayAsync(reader) { + const length = this.readInt32() + + if (length === -1) { + return [] + } + + const array = new Array(length) + for (let i = 0; i < length; i++) { + array[i] = await reader(this) + } + + return array + } + + readVarInt() { + let currentByte + let result = 0 + let i = 0 + + do { + currentByte = this.buffer[this.offset++] + result += (currentByte & OTHER_BITS) << i + i += 7 + } while (currentByte >= MOST_SIGNIFICANT_BIT) + + return Decoder.decodeZigZag(result) + } + + // By default JavaScript's numbers are of type float64, performing bitwise operations converts the numbers to a signed 32-bit integer + // Unsigned Right Shift Operator >>> ensures the returned value is an unsigned 32-bit integer + readUVarInt() { + let currentByte + let result = 0 + let i = 0 + while (((currentByte = this.buffer[this.offset++]) & MOST_SIGNIFICANT_BIT) !== 0) { + result |= (currentByte & OTHER_BITS) << i + i += 7 + if (i > 28) { + throw new KafkaJSInvalidVarIntError('Invalid VarInt, must contain 5 bytes or less') + } + } + result |= currentByte << i + return result >>> 0 + } + + readTaggedFields() { + const numberOfTaggedFields = this.readUVarInt() + + if (numberOfTaggedFields === 0) { + return null + } + + const taggedFields = {} + + for (let i = 0; i < numberOfTaggedFields; i++) { + // Right now this will read tag, the field length, and then length number of bytes for the field value skipping over the tag + this.readUVarInt() + this.readUVarIntBytes() + } + + return taggedFields + } + + readVarLong() { + let currentByte + let result = Long.fromInt(0) + let i = 0 + + do { + if (i > 63) { + throw new KafkaJSInvalidLongError('Invalid Long, must contain 9 bytes or less') + } + currentByte = this.buffer[this.offset++] + result = result.add(Long.fromInt(currentByte & OTHER_BITS).shiftLeft(i)) + i += 7 + } while (currentByte >= MOST_SIGNIFICANT_BIT) + + return Decoder.decodeZigZag64(result) + } + + slice(size) { + return new Decoder(this.buffer.slice(this.offset, this.offset + size)) + } + + forward(size) { + this.offset += size + } +} diff --git a/node_modules/kafkajs/src/protocol/encoder.js b/node_modules/kafkajs/src/protocol/encoder.js new file mode 100644 index 0000000..290bb97 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/encoder.js @@ -0,0 +1,405 @@ +const Long = require('../utils/long') + +const INT8_SIZE = 1 +const INT16_SIZE = 2 +const INT32_SIZE = 4 +const INT64_SIZE = 8 +const DOUBLE_SIZE = 8 + +const MOST_SIGNIFICANT_BIT = 0x80 // 128 +const OTHER_BITS = 0x7f // 127 +const UNSIGNED_INT32_MAX_NUMBER = 0xffffff80 +const UNSIGNED_INT64_MAX_NUMBER = 0xffffffffffffff80n + +module.exports = class Encoder { + static encodeZigZag(value) { + return (value << 1) ^ (value >> 31) + } + + static encodeZigZag64(value) { + const longValue = Long.fromValue(value) + return longValue.shiftLeft(1).xor(longValue.shiftRight(63)) + } + + static sizeOfVarInt(value) { + let encodedValue = this.encodeZigZag(value) + let bytes = 1 + + while ((encodedValue & UNSIGNED_INT32_MAX_NUMBER) !== 0) { + bytes += 1 + encodedValue >>>= 7 + } + + return bytes + } + + static sizeOfVarLong(value) { + let longValue = Encoder.encodeZigZag64(value) + let bytes = 1 + + while (longValue.and(UNSIGNED_INT64_MAX_NUMBER).notEquals(Long.fromInt(0))) { + bytes += 1 + longValue = longValue.shiftRightUnsigned(7) + } + + return bytes + } + + static sizeOfVarIntBytes(value) { + const size = value == null ? -1 : Buffer.byteLength(value) + + if (size < 0) { + return Encoder.sizeOfVarInt(-1) + } + + return Encoder.sizeOfVarInt(size) + size + } + + static nextPowerOfTwo(value) { + return 1 << (31 - Math.clz32(value) + 1) + } + + /** + * Construct a new encoder with the given initial size + * + * @param {number} [initialSize] initial size + */ + constructor(initialSize = 511) { + this.buf = Buffer.alloc(Encoder.nextPowerOfTwo(initialSize)) + this.offset = 0 + } + + /** + * @param {Buffer} buffer + */ + writeBufferInternal(buffer) { + const bufferLength = buffer.length + this.ensureAvailable(bufferLength) + buffer.copy(this.buf, this.offset, 0) + this.offset += bufferLength + } + + ensureAvailable(length) { + if (this.offset + length > this.buf.length) { + const newLength = Encoder.nextPowerOfTwo(this.offset + length) + const newBuffer = Buffer.alloc(newLength) + this.buf.copy(newBuffer, 0, 0, this.offset) + this.buf = newBuffer + } + } + + get buffer() { + return this.buf.slice(0, this.offset) + } + + writeInt8(value) { + this.ensureAvailable(INT8_SIZE) + this.buf.writeInt8(value, this.offset) + this.offset += INT8_SIZE + return this + } + + writeInt16(value) { + this.ensureAvailable(INT16_SIZE) + this.buf.writeInt16BE(value, this.offset) + this.offset += INT16_SIZE + return this + } + + writeInt32(value) { + this.ensureAvailable(INT32_SIZE) + this.buf.writeInt32BE(value, this.offset) + this.offset += INT32_SIZE + return this + } + + writeUInt32(value) { + this.ensureAvailable(INT32_SIZE) + this.buf.writeUInt32BE(value, this.offset) + this.offset += INT32_SIZE + return this + } + + writeInt64(value) { + this.ensureAvailable(INT64_SIZE) + const longValue = Long.fromValue(value) + this.buf.writeInt32BE(longValue.getHighBits(), this.offset) + this.buf.writeInt32BE(longValue.getLowBits(), this.offset + INT32_SIZE) + this.offset += INT64_SIZE + return this + } + + writeDouble(value) { + this.ensureAvailable(DOUBLE_SIZE) + this.buf.writeDoubleBE(value, this.offset) + this.offset += DOUBLE_SIZE + return this + } + + writeBoolean(value) { + value ? this.writeInt8(1) : this.writeInt8(0) + return this + } + + writeString(value) { + if (value == null) { + this.writeInt16(-1) + return this + } + + const byteLength = Buffer.byteLength(value, 'utf8') + this.ensureAvailable(INT16_SIZE + byteLength) + this.writeInt16(byteLength) + this.buf.write(value, this.offset, byteLength, 'utf8') + this.offset += byteLength + return this + } + + writeVarIntString(value) { + if (value == null) { + this.writeVarInt(-1) + return this + } + + const byteLength = Buffer.byteLength(value, 'utf8') + this.writeVarInt(byteLength) + this.ensureAvailable(byteLength) + this.buf.write(value, this.offset, byteLength, 'utf8') + this.offset += byteLength + return this + } + + writeUVarIntString(value) { + if (value == null) { + this.writeUVarInt(0) + return this + } + + const byteLength = Buffer.byteLength(value, 'utf8') + this.writeUVarInt(byteLength + 1) + this.ensureAvailable(byteLength) + this.buf.write(value, this.offset, byteLength, 'utf8') + this.offset += byteLength + return this + } + + writeBytes(value) { + if (value == null) { + this.writeInt32(-1) + return this + } + + if (Buffer.isBuffer(value)) { + // raw bytes + this.ensureAvailable(INT32_SIZE + value.length) + this.writeInt32(value.length) + this.writeBufferInternal(value) + } else { + const valueToWrite = String(value) + const byteLength = Buffer.byteLength(valueToWrite, 'utf8') + this.ensureAvailable(INT32_SIZE + byteLength) + this.writeInt32(byteLength) + this.buf.write(valueToWrite, this.offset, byteLength, 'utf8') + this.offset += byteLength + } + + return this + } + + writeVarIntBytes(value) { + if (value == null) { + this.writeVarInt(-1) + return this + } + + if (Buffer.isBuffer(value)) { + // raw bytes + this.writeVarInt(value.length) + this.writeBufferInternal(value) + } else { + const valueToWrite = String(value) + const byteLength = Buffer.byteLength(valueToWrite, 'utf8') + this.writeVarInt(byteLength) + this.ensureAvailable(byteLength) + this.buf.write(valueToWrite, this.offset, byteLength, 'utf8') + this.offset += byteLength + } + + return this + } + + writeUVarIntBytes(value) { + if (value == null) { + this.writeVarInt(0) + return this + } + + if (Buffer.isBuffer(value)) { + // raw bytes + this.writeUVarInt(value.length + 1) + this.writeBufferInternal(value) + } else { + const valueToWrite = String(value) + const byteLength = Buffer.byteLength(valueToWrite, 'utf8') + this.writeUVarInt(byteLength + 1) + this.ensureAvailable(byteLength) + this.buf.write(valueToWrite, this.offset, byteLength, 'utf8') + this.offset += byteLength + } + + return this + } + + writeEncoder(value) { + if (value == null || !Buffer.isBuffer(value.buf)) { + throw new Error('value should be an instance of Encoder') + } + + this.writeBufferInternal(value.buffer) + return this + } + + writeEncoderArray(value) { + if (!Array.isArray(value) || value.some(v => v == null || !Buffer.isBuffer(v.buf))) { + throw new Error('all values should be an instance of Encoder[]') + } + + value.forEach(v => { + this.writeBufferInternal(v.buffer) + }) + return this + } + + writeBuffer(value) { + if (!Buffer.isBuffer(value)) { + throw new Error('value should be an instance of Buffer') + } + + this.writeBufferInternal(value) + return this + } + + /** + * @param {any[]} array + * @param {'int32'|'number'|'string'|'object'} [type] + */ + writeNullableArray(array, type) { + // A null value is encoded with length of -1 and there are no following bytes + // On the context of this library, empty array and null are the same thing + const length = array.length !== 0 ? array.length : -1 + this.writeArray(array, type, length) + return this + } + + /** + * @param {any[]} array + * @param {'int32'|'number'|'string'|'object'} [type] + * @param {number} [length] + */ + writeArray(array, type, length) { + const arrayLength = length == null ? array.length : length + this.writeInt32(arrayLength) + if (type !== undefined) { + switch (type) { + case 'int32': + case 'number': + array.forEach(value => this.writeInt32(value)) + break + case 'string': + array.forEach(value => this.writeString(value)) + break + case 'object': + this.writeEncoderArray(array) + break + } + } else { + array.forEach(value => { + switch (typeof value) { + case 'number': + this.writeInt32(value) + break + case 'string': + this.writeString(value) + break + case 'object': + this.writeEncoder(value) + break + } + }) + } + return this + } + + writeVarIntArray(array, type) { + if (type === 'object') { + this.writeVarInt(array.length) + this.writeEncoderArray(array) + } else { + const objectArray = array.filter(v => typeof v === 'object') + this.writeVarInt(objectArray.length) + this.writeEncoderArray(objectArray) + } + return this + } + + writeUVarIntArray(array, type) { + if (type === 'object') { + this.writeUVarInt(array.length + 1) + this.writeEncoderArray(array) + } else if (array === null) { + this.writeUVarInt(0) + } else { + const objectArray = array.filter(v => typeof v === 'object') + this.writeUVarInt(objectArray.length + 1) + this.writeEncoderArray(objectArray) + } + return this + } + + // Based on: + // https://en.wikipedia.org/wiki/LEB128 Using LEB128 format similar to VLQ. + // https://github.com/addthis/stream-lib/blob/master/src/main/java/com/clearspring/analytics/util/Varint.java#L106 + writeVarInt(value) { + return this.writeUVarInt(Encoder.encodeZigZag(value)) + } + + writeUVarInt(value) { + const byteArray = [] + while ((value & UNSIGNED_INT32_MAX_NUMBER) !== 0) { + byteArray.push((value & OTHER_BITS) | MOST_SIGNIFICANT_BIT) + value >>>= 7 + } + byteArray.push(value & OTHER_BITS) + this.writeBufferInternal(Buffer.from(byteArray)) + return this + } + + writeVarLong(value) { + const byteArray = [] + let longValue = Encoder.encodeZigZag64(value) + + while (longValue.and(UNSIGNED_INT64_MAX_NUMBER).notEquals(Long.fromInt(0))) { + byteArray.push( + longValue + .and(OTHER_BITS) + .or(MOST_SIGNIFICANT_BIT) + .toInt() + ) + longValue = longValue.shiftRightUnsigned(7) + } + + byteArray.push(longValue.toInt()) + + this.writeBufferInternal(Buffer.from(byteArray)) + return this + } + + size() { + // We can use the offset here directly, because we anyways will not re-encode the buffer when writing + return this.offset + } + + toJSON() { + return this.buffer.toJSON() + } +} diff --git a/node_modules/kafkajs/src/protocol/error.js b/node_modules/kafkajs/src/protocol/error.js new file mode 100644 index 0000000..c070cb9 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/error.js @@ -0,0 +1,601 @@ +const { KafkaJSProtocolError } = require('../errors') +const websiteUrl = require('../utils/websiteUrl') + +const errorCodes = [ + { + type: 'UNKNOWN', + code: -1, + retriable: false, + message: 'The server experienced an unexpected error when processing the request', + }, + { + type: 'OFFSET_OUT_OF_RANGE', + code: 1, + retriable: false, + message: 'The requested offset is not within the range of offsets maintained by the server', + }, + { + type: 'CORRUPT_MESSAGE', + code: 2, + retriable: true, + message: + 'This message has failed its CRC checksum, exceeds the valid size, or is otherwise corrupt', + }, + { + type: 'UNKNOWN_TOPIC_OR_PARTITION', + code: 3, + retriable: true, + message: 'This server does not host this topic-partition', + }, + { + type: 'INVALID_FETCH_SIZE', + code: 4, + retriable: false, + message: 'The requested fetch size is invalid', + }, + { + type: 'LEADER_NOT_AVAILABLE', + code: 5, + retriable: true, + message: + 'There is no leader for this topic-partition as we are in the middle of a leadership election', + }, + { + type: 'NOT_LEADER_FOR_PARTITION', + code: 6, + retriable: true, + message: 'This server is not the leader for that topic-partition', + }, + { + type: 'REQUEST_TIMED_OUT', + code: 7, + retriable: true, + message: 'The request timed out', + }, + { + type: 'BROKER_NOT_AVAILABLE', + code: 8, + retriable: false, + message: 'The broker is not available', + }, + { + type: 'REPLICA_NOT_AVAILABLE', + code: 9, + retriable: true, + message: 'The replica is not available for the requested topic-partition', + }, + { + type: 'MESSAGE_TOO_LARGE', + code: 10, + retriable: false, + message: + 'The request included a message larger than the max message size the server will accept', + }, + { + type: 'STALE_CONTROLLER_EPOCH', + code: 11, + retriable: false, + message: 'The controller moved to another broker', + }, + { + type: 'OFFSET_METADATA_TOO_LARGE', + code: 12, + retriable: false, + message: 'The metadata field of the offset request was too large', + }, + { + type: 'NETWORK_EXCEPTION', + code: 13, + retriable: true, + message: 'The server disconnected before a response was received', + }, + { + type: 'GROUP_LOAD_IN_PROGRESS', + code: 14, + retriable: true, + message: "The coordinator is loading and hence can't process requests for this group", + }, + { + type: 'GROUP_COORDINATOR_NOT_AVAILABLE', + code: 15, + retriable: true, + message: 'The group coordinator is not available', + }, + { + type: 'NOT_COORDINATOR_FOR_GROUP', + code: 16, + retriable: true, + message: 'This is not the correct coordinator for this group', + }, + { + type: 'INVALID_TOPIC_EXCEPTION', + code: 17, + retriable: false, + message: 'The request attempted to perform an operation on an invalid topic', + }, + { + type: 'RECORD_LIST_TOO_LARGE', + code: 18, + retriable: false, + message: + 'The request included message batch larger than the configured segment size on the server', + }, + { + type: 'NOT_ENOUGH_REPLICAS', + code: 19, + retriable: true, + message: 'Messages are rejected since there are fewer in-sync replicas than required', + }, + { + type: 'NOT_ENOUGH_REPLICAS_AFTER_APPEND', + code: 20, + retriable: true, + message: 'Messages are written to the log, but to fewer in-sync replicas than required', + }, + { + type: 'INVALID_REQUIRED_ACKS', + code: 21, + retriable: false, + message: 'Produce request specified an invalid value for required acks', + }, + { + type: 'ILLEGAL_GENERATION', + code: 22, + retriable: false, + message: 'Specified group generation id is not valid', + }, + { + type: 'INCONSISTENT_GROUP_PROTOCOL', + code: 23, + retriable: false, + message: + "The group member's supported protocols are incompatible with those of existing members", + }, + { + type: 'INVALID_GROUP_ID', + code: 24, + retriable: false, + message: 'The configured groupId is invalid', + }, + { + type: 'UNKNOWN_MEMBER_ID', + code: 25, + retriable: false, + message: 'The coordinator is not aware of this member', + }, + { + type: 'INVALID_SESSION_TIMEOUT', + code: 26, + retriable: false, + message: + 'The session timeout is not within the range allowed by the broker (as configured by group.min.session.timeout.ms and group.max.session.timeout.ms)', + }, + { + type: 'REBALANCE_IN_PROGRESS', + code: 27, + retriable: false, + message: 'The group is rebalancing, so a rejoin is needed', + helpUrl: websiteUrl('docs/faq', 'what-does-it-mean-to-get-rebalance-in-progress-errors'), + }, + { + type: 'INVALID_COMMIT_OFFSET_SIZE', + code: 28, + retriable: false, + message: 'The committing offset data size is not valid', + }, + { + type: 'TOPIC_AUTHORIZATION_FAILED', + code: 29, + retriable: false, + message: 'Not authorized to access topics: [Topic authorization failed]', + }, + { + type: 'GROUP_AUTHORIZATION_FAILED', + code: 30, + retriable: false, + message: 'Not authorized to access group: Group authorization failed', + }, + { + type: 'CLUSTER_AUTHORIZATION_FAILED', + code: 31, + retriable: false, + message: 'Cluster authorization failed', + }, + { + type: 'INVALID_TIMESTAMP', + code: 32, + retriable: false, + message: 'The timestamp of the message is out of acceptable range', + }, + { + type: 'UNSUPPORTED_SASL_MECHANISM', + code: 33, + retriable: false, + message: 'The broker does not support the requested SASL mechanism', + }, + { + type: 'ILLEGAL_SASL_STATE', + code: 34, + retriable: false, + message: 'Request is not valid given the current SASL state', + }, + { + type: 'UNSUPPORTED_VERSION', + code: 35, + retriable: false, + message: 'The version of API is not supported', + }, + { + type: 'TOPIC_ALREADY_EXISTS', + code: 36, + retriable: false, + message: 'Topic with this name already exists', + }, + { + type: 'INVALID_PARTITIONS', + code: 37, + retriable: false, + message: 'Number of partitions is invalid', + }, + { + type: 'INVALID_REPLICATION_FACTOR', + code: 38, + retriable: false, + message: 'Replication-factor is invalid', + }, + { + type: 'INVALID_REPLICA_ASSIGNMENT', + code: 39, + retriable: false, + message: 'Replica assignment is invalid', + }, + { + type: 'INVALID_CONFIG', + code: 40, + retriable: false, + message: 'Configuration is invalid', + }, + { + type: 'NOT_CONTROLLER', + code: 41, + retriable: true, + message: 'This is not the correct controller for this cluster', + }, + { + type: 'INVALID_REQUEST', + code: 42, + retriable: false, + message: + 'This most likely occurs because of a request being malformed by the client library or the message was sent to an incompatible broker. See the broker logs for more details', + }, + { + type: 'UNSUPPORTED_FOR_MESSAGE_FORMAT', + code: 43, + retriable: false, + message: 'The message format version on the broker does not support the request', + }, + { + type: 'POLICY_VIOLATION', + code: 44, + retriable: false, + message: 'Request parameters do not satisfy the configured policy', + }, + { + type: 'OUT_OF_ORDER_SEQUENCE_NUMBER', + code: 45, + retriable: false, + message: 'The broker received an out of order sequence number', + }, + { + type: 'DUPLICATE_SEQUENCE_NUMBER', + code: 46, + retriable: false, + message: 'The broker received a duplicate sequence number', + }, + { + type: 'INVALID_PRODUCER_EPOCH', + code: 47, + retriable: false, + message: + "Producer attempted an operation with an old epoch. Either there is a newer producer with the same transactionalId, or the producer's transaction has been expired by the broker", + }, + { + type: 'INVALID_TXN_STATE', + code: 48, + retriable: false, + message: 'The producer attempted a transactional operation in an invalid state', + }, + { + type: 'INVALID_PRODUCER_ID_MAPPING', + code: 49, + retriable: false, + message: + 'The producer attempted to use a producer id which is not currently assigned to its transactional id', + }, + { + type: 'INVALID_TRANSACTION_TIMEOUT', + code: 50, + retriable: false, + message: + 'The transaction timeout is larger than the maximum value allowed by the broker (as configured by max.transaction.timeout.ms)', + }, + { + type: 'CONCURRENT_TRANSACTIONS', + code: 51, + /** + * The concurrent transactions error has "retriable" set to false on the protocol documentation (https://kafka.apache.org/protocol.html#protocol_error_codes) + * but the server expects the clients to retry. PR #223 + * @see https://github.com/apache/kafka/blob/12f310d50e7f5b1c18c4f61a119a6cd830da3bc0/core/src/main/scala/kafka/coordinator/transaction/TransactionCoordinator.scala#L153 + */ + retriable: true, + message: + 'The producer attempted to update a transaction while another concurrent operation on the same transaction was ongoing', + }, + { + type: 'TRANSACTION_COORDINATOR_FENCED', + code: 52, + retriable: false, + message: + 'Indicates that the transaction coordinator sending a WriteTxnMarker is no longer the current coordinator for a given producer', + }, + { + type: 'TRANSACTIONAL_ID_AUTHORIZATION_FAILED', + code: 53, + retriable: false, + message: 'Transactional Id authorization failed', + }, + { + type: 'SECURITY_DISABLED', + code: 54, + retriable: false, + message: 'Security features are disabled', + }, + { + type: 'OPERATION_NOT_ATTEMPTED', + code: 55, + retriable: false, + message: + 'The broker did not attempt to execute this operation. This may happen for batched RPCs where some operations in the batch failed, causing the broker to respond without trying the rest', + }, + { + type: 'KAFKA_STORAGE_ERROR', + code: 56, + retriable: true, + message: 'Disk error when trying to access log file on the disk', + }, + { + type: 'LOG_DIR_NOT_FOUND', + code: 57, + retriable: false, + message: 'The user-specified log directory is not found in the broker config', + }, + { + type: 'SASL_AUTHENTICATION_FAILED', + code: 58, + retriable: false, + message: 'SASL Authentication failed', + helpUrl: websiteUrl('docs/configuration', 'sasl'), + }, + { + type: 'UNKNOWN_PRODUCER_ID', + code: 59, + retriable: false, + message: + "This exception is raised by the broker if it could not locate the producer metadata associated with the producerId in question. This could happen if, for instance, the producer's records were deleted because their retention time had elapsed. Once the last records of the producerId are removed, the producer's metadata is removed from the broker, and future appends by the producer will return this exception", + }, + { + type: 'REASSIGNMENT_IN_PROGRESS', + code: 60, + retriable: false, + message: 'A partition reassignment is in progress', + }, + { + type: 'DELEGATION_TOKEN_AUTH_DISABLED', + code: 61, + retriable: false, + message: 'Delegation Token feature is not enabled', + }, + { + type: 'DELEGATION_TOKEN_NOT_FOUND', + code: 62, + retriable: false, + message: 'Delegation Token is not found on server', + }, + { + type: 'DELEGATION_TOKEN_OWNER_MISMATCH', + code: 63, + retriable: false, + message: 'Specified Principal is not valid Owner/Renewer', + }, + { + type: 'DELEGATION_TOKEN_REQUEST_NOT_ALLOWED', + code: 64, + retriable: false, + message: + 'Delegation Token requests are not allowed on PLAINTEXT/1-way SSL channels and on delegation token authenticated channels', + }, + { + type: 'DELEGATION_TOKEN_AUTHORIZATION_FAILED', + code: 65, + retriable: false, + message: 'Delegation Token authorization failed', + }, + { + type: 'DELEGATION_TOKEN_EXPIRED', + code: 66, + retriable: false, + message: 'Delegation Token is expired', + }, + { + type: 'INVALID_PRINCIPAL_TYPE', + code: 67, + retriable: false, + message: 'Supplied principalType is not supported', + }, + { + type: 'NON_EMPTY_GROUP', + code: 68, + retriable: false, + message: 'The group is not empty', + }, + { + type: 'GROUP_ID_NOT_FOUND', + code: 69, + retriable: false, + message: 'The group id was not found', + }, + { + type: 'FETCH_SESSION_ID_NOT_FOUND', + code: 70, + retriable: true, + message: 'The fetch session ID was not found', + }, + { + type: 'INVALID_FETCH_SESSION_EPOCH', + code: 71, + retriable: true, + message: 'The fetch session epoch is invalid', + }, + { + type: 'LISTENER_NOT_FOUND', + code: 72, + retriable: true, + message: + 'There is no listener on the leader broker that matches the listener on which metadata request was processed', + }, + { + type: 'TOPIC_DELETION_DISABLED', + code: 73, + retriable: false, + message: 'Topic deletion is disabled', + }, + { + type: 'FENCED_LEADER_EPOCH', + code: 74, + retriable: true, + message: 'The leader epoch in the request is older than the epoch on the broker', + }, + { + type: 'UNKNOWN_LEADER_EPOCH', + code: 75, + retriable: true, + message: 'The leader epoch in the request is newer than the epoch on the broker', + }, + { + type: 'UNSUPPORTED_COMPRESSION_TYPE', + code: 76, + retriable: false, + message: 'The requesting client does not support the compression type of given partition', + }, + { + type: 'STALE_BROKER_EPOCH', + code: 77, + retriable: false, + message: 'Broker epoch has changed', + }, + { + type: 'OFFSET_NOT_AVAILABLE', + code: 78, + retriable: true, + message: + 'The leader high watermark has not caught up from a recent leader election so the offsets cannot be guaranteed to be monotonically increasing', + }, + { + type: 'MEMBER_ID_REQUIRED', + code: 79, + retriable: false, + message: + 'The group member needs to have a valid member id before actually entering a consumer group', + }, + { + type: 'PREFERRED_LEADER_NOT_AVAILABLE', + code: 80, + retriable: true, + message: 'The preferred leader was not available', + }, + { + type: 'GROUP_MAX_SIZE_REACHED', + code: 81, + retriable: false, + message: + 'The consumer group has reached its max size. It already has the configured maximum number of members', + }, + { + type: 'FENCED_INSTANCE_ID', + code: 82, + retriable: false, + message: + 'The broker rejected this static consumer since another consumer with the same group instance id has registered with a different member id', + }, + { + type: 'ELIGIBLE_LEADERS_NOT_AVAILABLE', + code: 83, + retriable: true, + message: 'Eligible topic partition leaders are not available', + }, + { + type: 'ELECTION_NOT_NEEDED', + code: 84, + retriable: true, + message: 'Leader election not needed for topic partition', + }, + { + type: 'NO_REASSIGNMENT_IN_PROGRESS', + code: 85, + retriable: false, + message: 'No partition reassignment is in progress', + }, + { + type: 'GROUP_SUBSCRIBED_TO_TOPIC', + code: 86, + retriable: false, + message: + 'Deleting offsets of a topic is forbidden while the consumer group is actively subscribed to it', + }, + { + type: 'INVALID_RECORD', + code: 87, + retriable: false, + message: 'This record has failed the validation on broker and hence be rejected', + }, + { + type: 'UNSTABLE_OFFSET_COMMIT', + code: 88, + retriable: true, + message: 'There are unstable offsets that need to be cleared', + }, +] + +const unknownErrorCode = errorCode => ({ + type: 'KAFKAJS_UNKNOWN_ERROR_CODE', + code: -99, + retriable: false, + message: `Unknown error code ${errorCode}`, +}) + +const SUCCESS_CODE = 0 +const UNSUPPORTED_VERSION_CODE = 35 + +const failure = code => code !== SUCCESS_CODE +const createErrorFromCode = code => { + return new KafkaJSProtocolError(errorCodes.find(e => e.code === code) || unknownErrorCode(code)) +} + +const failIfVersionNotSupported = code => { + if (code === UNSUPPORTED_VERSION_CODE) { + throw createErrorFromCode(UNSUPPORTED_VERSION_CODE) + } +} + +const staleMetadata = e => + ['UNKNOWN_TOPIC_OR_PARTITION', 'LEADER_NOT_AVAILABLE', 'NOT_LEADER_FOR_PARTITION'].includes( + e.type + ) + +module.exports = { + failure, + errorCodes, + createErrorFromCode, + failIfVersionNotSupported, + staleMetadata, +} diff --git a/node_modules/kafkajs/src/protocol/isolationLevel.js b/node_modules/kafkajs/src/protocol/isolationLevel.js new file mode 100644 index 0000000..a80b19e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/isolationLevel.js @@ -0,0 +1,15 @@ +/** + * Enum for isolation levels + * @readonly + * @enum {number} + */ +module.exports = { + // Makes all records visible + READ_UNCOMMITTED: 0, + + // non-transactional and COMMITTED transactional records are visible. It returns all data + // from offsets smaller than the current LSO (last stable offset), and enables the inclusion of + // the list of aborted transactions in the result, which allows consumers to discard ABORTED + // transactional records + READ_COMMITTED: 1, +} diff --git a/node_modules/kafkajs/src/protocol/message/compression/gzip.js b/node_modules/kafkajs/src/protocol/message/compression/gzip.js new file mode 100644 index 0000000..ee7cf0f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/message/compression/gzip.js @@ -0,0 +1,23 @@ +const { promisify } = require('util') +const zlib = require('zlib') + +const gzip = promisify(zlib.gzip) +const unzip = promisify(zlib.unzip) + +module.exports = { + /** + * @param {Encoder} encoder + * @returns {Promise} + */ + async compress(encoder) { + return await gzip(encoder.buffer) + }, + + /** + * @param {Buffer} buffer + * @returns {Promise} + */ + async decompress(buffer) { + return await unzip(buffer) + }, +} diff --git a/node_modules/kafkajs/src/protocol/message/compression/index.js b/node_modules/kafkajs/src/protocol/message/compression/index.js new file mode 100644 index 0000000..c3e24c4 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/message/compression/index.js @@ -0,0 +1,38 @@ +const { KafkaJSNotImplemented } = require('../../../errors') + +const COMPRESSION_CODEC_MASK = 0x07 + +const Types = { + None: 0, + GZIP: 1, + Snappy: 2, + LZ4: 3, + ZSTD: 4, +} + +const Codecs = { + [Types.GZIP]: () => require('./gzip'), + [Types.Snappy]: () => { + throw new KafkaJSNotImplemented('Snappy compression not implemented') + }, + [Types.LZ4]: () => { + throw new KafkaJSNotImplemented('LZ4 compression not implemented') + }, + [Types.ZSTD]: () => { + throw new KafkaJSNotImplemented('ZSTD compression not implemented') + }, +} + +const lookupCodec = type => (Codecs[type] ? Codecs[type]() : null) +const lookupCodecByAttributes = attributes => { + const codec = Codecs[attributes & COMPRESSION_CODEC_MASK] + return codec ? codec() : null +} + +module.exports = { + Types, + Codecs, + lookupCodec, + lookupCodecByAttributes, + COMPRESSION_CODEC_MASK, +} diff --git a/node_modules/kafkajs/src/protocol/message/decoder.js b/node_modules/kafkajs/src/protocol/message/decoder.js new file mode 100644 index 0000000..372fdb3 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/message/decoder.js @@ -0,0 +1,37 @@ +const { + KafkaJSPartialMessageError, + KafkaJSUnsupportedMagicByteInMessageSet, +} = require('../../errors') + +const V0Decoder = require('./v0/decoder') +const V1Decoder = require('./v1/decoder') + +const decodeMessage = (decoder, magicByte) => { + switch (magicByte) { + case 0: + return V0Decoder(decoder) + case 1: + return V1Decoder(decoder) + default: + throw new KafkaJSUnsupportedMagicByteInMessageSet( + `Unsupported MessageSet message version, magic byte: ${magicByte}` + ) + } +} + +module.exports = (offset, size, decoder) => { + // Don't decrement decoder.offset because slice is already considering the current + // offset of the decoder + const remainingBytes = Buffer.byteLength(decoder.slice(size).buffer) + + if (remainingBytes < size) { + throw new KafkaJSPartialMessageError( + `Tried to decode a partial message: remainingBytes(${remainingBytes}) < messageSize(${size})` + ) + } + + const crc = decoder.readInt32() + const magicByte = decoder.readInt8() + const message = decodeMessage(decoder, magicByte) + return Object.assign({ offset, size, crc, magicByte }, message) +} diff --git a/node_modules/kafkajs/src/protocol/message/index.js b/node_modules/kafkajs/src/protocol/message/index.js new file mode 100644 index 0000000..05beb90 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/message/index.js @@ -0,0 +1,6 @@ +const versions = { + 0: require('./v0'), + 1: require('./v1'), +} + +module.exports = ({ version = 0 }) => versions[version] diff --git a/node_modules/kafkajs/src/protocol/message/v0/decoder.js b/node_modules/kafkajs/src/protocol/message/v0/decoder.js new file mode 100644 index 0000000..20436b4 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/message/v0/decoder.js @@ -0,0 +1,5 @@ +module.exports = decoder => ({ + attributes: decoder.readInt8(), + key: decoder.readBytes(), + value: decoder.readBytes(), +}) diff --git a/node_modules/kafkajs/src/protocol/message/v0/index.js b/node_modules/kafkajs/src/protocol/message/v0/index.js new file mode 100644 index 0000000..6ff38bb --- /dev/null +++ b/node_modules/kafkajs/src/protocol/message/v0/index.js @@ -0,0 +1,24 @@ +const Encoder = require('../../encoder') +const crc32 = require('../../crc32') +const { Types: Compression, COMPRESSION_CODEC_MASK } = require('../compression') + +/** + * v0 + * Message => Crc MagicByte Attributes Key Value + * Crc => int32 + * MagicByte => int8 + * Attributes => int8 + * Key => bytes + * Value => bytes + */ + +module.exports = ({ compression = Compression.None, key, value }) => { + const content = new Encoder() + .writeInt8(0) // magicByte + .writeInt8(compression & COMPRESSION_CODEC_MASK) + .writeBytes(key) + .writeBytes(value) + + const crc = crc32(content) + return new Encoder().writeInt32(crc).writeEncoder(content) +} diff --git a/node_modules/kafkajs/src/protocol/message/v1/decoder.js b/node_modules/kafkajs/src/protocol/message/v1/decoder.js new file mode 100644 index 0000000..67dff7e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/message/v1/decoder.js @@ -0,0 +1,6 @@ +module.exports = decoder => ({ + attributes: decoder.readInt8(), + timestamp: decoder.readInt64().toString(), + key: decoder.readBytes(), + value: decoder.readBytes(), +}) diff --git a/node_modules/kafkajs/src/protocol/message/v1/index.js b/node_modules/kafkajs/src/protocol/message/v1/index.js new file mode 100644 index 0000000..a322610 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/message/v1/index.js @@ -0,0 +1,26 @@ +const Encoder = require('../../encoder') +const crc32 = require('../../crc32') +const { Types: Compression, COMPRESSION_CODEC_MASK } = require('../compression') + +/** + * v1 (supported since 0.10.0) + * Message => Crc MagicByte Attributes Key Value + * Crc => int32 + * MagicByte => int8 + * Attributes => int8 + * Timestamp => int64 + * Key => bytes + * Value => bytes + */ + +module.exports = ({ compression = Compression.None, timestamp = Date.now(), key, value }) => { + const content = new Encoder() + .writeInt8(1) // magicByte + .writeInt8(compression & COMPRESSION_CODEC_MASK) + .writeInt64(timestamp) + .writeBytes(key) + .writeBytes(value) + + const crc = crc32(content) + return new Encoder().writeInt32(crc).writeEncoder(content) +} diff --git a/node_modules/kafkajs/src/protocol/messageSet/decoder.js b/node_modules/kafkajs/src/protocol/messageSet/decoder.js new file mode 100644 index 0000000..e90c9a9 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/messageSet/decoder.js @@ -0,0 +1,91 @@ +const Long = require('../../utils/long') +const Decoder = require('../decoder') +const MessageDecoder = require('../message/decoder') +const { lookupCodecByAttributes } = require('../message/compression') +const { KafkaJSPartialMessageError } = require('../../errors') + +/** + * MessageSet => [Offset MessageSize Message] + * Offset => int64 + * MessageSize => int32 + * Message => Bytes + */ + +module.exports = async (primaryDecoder, size = null) => { + const messages = [] + const messageSetSize = size || primaryDecoder.readInt32() + const messageSetDecoder = primaryDecoder.slice(messageSetSize) + + while (messageSetDecoder.offset < messageSetSize) { + try { + const message = EntryDecoder(messageSetDecoder) + const codec = lookupCodecByAttributes(message.attributes) + + if (codec) { + const buffer = await codec.decompress(message.value) + messages.push(...EntriesDecoder(new Decoder(buffer), message)) + } else { + messages.push(message) + } + } catch (e) { + if (e.name === 'KafkaJSPartialMessageError') { + // We tried to decode a partial message, it means that minBytes + // is probably too low + break + } + + if (e.name === 'KafkaJSUnsupportedMagicByteInMessageSet') { + // Received a MessageSet and a RecordBatch on the same response, the cluster is probably + // upgrading the message format from 0.10 to 0.11. Stop processing this message set to + // receive the full record batch on the next request + break + } + + throw e + } + } + + primaryDecoder.forward(messageSetSize) + return messages +} + +const EntriesDecoder = (decoder, compressedMessage) => { + const messages = [] + + while (decoder.offset < decoder.buffer.length) { + messages.push(EntryDecoder(decoder)) + } + + if (compressedMessage.magicByte > 0 && compressedMessage.offset >= 0) { + const compressedOffset = Long.fromValue(compressedMessage.offset) + const lastMessageOffset = Long.fromValue(messages[messages.length - 1].offset) + const baseOffset = compressedOffset - lastMessageOffset + + for (const message of messages) { + message.offset = Long.fromValue(message.offset) + .add(baseOffset) + .toString() + } + } + + return messages +} + +const EntryDecoder = decoder => { + if (!decoder.canReadInt64()) { + throw new KafkaJSPartialMessageError( + `Tried to decode a partial message: There isn't enough bytes to read the offset` + ) + } + + const offset = decoder.readInt64().toString() + + if (!decoder.canReadInt32()) { + throw new KafkaJSPartialMessageError( + `Tried to decode a partial message: There isn't enough bytes to read the message size` + ) + } + + const size = decoder.readInt32() + return MessageDecoder(offset, size, decoder) +} diff --git a/node_modules/kafkajs/src/protocol/messageSet/index.js b/node_modules/kafkajs/src/protocol/messageSet/index.js new file mode 100644 index 0000000..79a4ede --- /dev/null +++ b/node_modules/kafkajs/src/protocol/messageSet/index.js @@ -0,0 +1,41 @@ +const Encoder = require('../encoder') +const MessageProtocol = require('../message') +const { Types } = require('../message/compression') + +/** + * MessageSet => [Offset MessageSize Message] + * Offset => int64 + * MessageSize => int32 + * Message => Bytes + */ + +/** + * [ + * { key: "", value: "" }, + * { key: "", value: "" }, + * ] + */ +module.exports = ({ messageVersion = 0, compression, entries }) => { + const isCompressed = compression !== Types.None + const Message = MessageProtocol({ version: messageVersion }) + const encoder = new Encoder() + + // Messages in a message set are __not__ encoded as an array. + // They are written in sequence. + // https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets + + entries.forEach((entry, i) => { + const message = Message(entry) + + // This is the offset used in kafka as the log sequence number. + // When the producer is sending non compressed messages, it can set the offsets to anything + // When the producer is sending compressed messages, to avoid server side recompression, each compressed message + // should have offset starting from 0 and increasing by one for each inner message in the compressed message + encoder.writeInt64(isCompressed ? i : -1) + encoder.writeInt32(message.size()) + + encoder.writeEncoder(message) + }) + + return encoder +} diff --git a/node_modules/kafkajs/src/protocol/recordBatch/crc32C/crc32C.js b/node_modules/kafkajs/src/protocol/recordBatch/crc32C/crc32C.js new file mode 100644 index 0000000..b3ab335 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/recordBatch/crc32C/crc32C.js @@ -0,0 +1,85 @@ +/** + * A javascript implementation of the CRC32 checksum that uses + * the CRC32-C polynomial, the same polynomial used by iSCSI + * + * also known as CRC32 Castagnoli + * based on: https://github.com/ashi009/node-fast-crc32c/blob/master/impls/js_crc32c.js + */ +const crc32C = buffer => { + let crc = 0 ^ -1 + for (let i = 0; i < buffer.length; i++) { + crc = T[(crc ^ buffer[i]) & 0xff] ^ (crc >>> 8) + } + + return (crc ^ -1) >>> 0 +} + +module.exports = crc32C + +// prettier-ignore +var T = new Int32Array([ + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, + 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, + 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, + 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, + 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, + 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, + 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, + 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, + 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, + 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, + 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, + 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, + 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, + 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, + 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, + 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, + 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, + 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, + 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, + 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 +]); diff --git a/node_modules/kafkajs/src/protocol/recordBatch/crc32C/index.js b/node_modules/kafkajs/src/protocol/recordBatch/crc32C/index.js new file mode 100644 index 0000000..eafb0a1 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/recordBatch/crc32C/index.js @@ -0,0 +1,4 @@ +const crc32C = require('./crc32C') +const unsigned = value => Uint32Array.from([value])[0] + +module.exports = buffer => unsigned(crc32C(buffer)) diff --git a/node_modules/kafkajs/src/protocol/recordBatch/header/v0/decoder.js b/node_modules/kafkajs/src/protocol/recordBatch/header/v0/decoder.js new file mode 100644 index 0000000..a1f56d5 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/recordBatch/header/v0/decoder.js @@ -0,0 +1,4 @@ +module.exports = decoder => ({ + key: decoder.readVarIntString(), + value: decoder.readVarIntBytes(), +}) diff --git a/node_modules/kafkajs/src/protocol/recordBatch/header/v0/index.js b/node_modules/kafkajs/src/protocol/recordBatch/header/v0/index.js new file mode 100644 index 0000000..1a81796 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/recordBatch/header/v0/index.js @@ -0,0 +1,12 @@ +const Encoder = require('../../../encoder') + +/** + * v0 + * Header => Key Value + * Key => varInt|string + * Value => varInt|bytes + */ + +module.exports = ({ key, value }) => { + return new Encoder().writeVarIntString(key).writeVarIntBytes(value) +} diff --git a/node_modules/kafkajs/src/protocol/recordBatch/record/v0/decoder.js b/node_modules/kafkajs/src/protocol/recordBatch/record/v0/decoder.js new file mode 100644 index 0000000..641fdd2 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/recordBatch/record/v0/decoder.js @@ -0,0 +1,69 @@ +const Long = require('../../../../utils/long') +const HeaderDecoder = require('../../header/v0/decoder') +const TimestampTypes = require('../../../timestampTypes') + +/** + * v0 + * Record => + * Length => Varint + * Attributes => Int8 + * TimestampDelta => Varlong + * OffsetDelta => Varint + * Key => varInt|Bytes + * Value => varInt|Bytes + * Headers => [HeaderKey HeaderValue] + * HeaderKey => VarInt|String + * HeaderValue => VarInt|Bytes + */ + +module.exports = (decoder, batchContext = {}) => { + const { + firstOffset, + firstTimestamp, + magicByte, + isControlBatch = false, + timestampType, + maxTimestamp, + } = batchContext + const attributes = decoder.readInt8() + + const timestampDelta = decoder.readVarLong() + const timestamp = + timestampType === TimestampTypes.LOG_APPEND_TIME && maxTimestamp + ? maxTimestamp + : Long.fromValue(firstTimestamp) + .add(timestampDelta) + .toString() + + const offsetDelta = decoder.readVarInt() + const offset = Long.fromValue(firstOffset) + .add(offsetDelta) + .toString() + + const key = decoder.readVarIntBytes() + const value = decoder.readVarIntBytes() + const headers = decoder.readVarIntArray(HeaderDecoder).reduce( + (obj, { key, value }) => ({ + ...obj, + [key]: + obj[key] === undefined + ? value + : Array.isArray(obj[key]) + ? obj[key].concat([value]) + : [obj[key], value], + }), + {} + ) + + return { + magicByte, + attributes, // Record level attributes are presently unused + timestamp, + offset, + key, + value, + headers, + isControlRecord: isControlBatch, + batchContext, + } +} diff --git a/node_modules/kafkajs/src/protocol/recordBatch/record/v0/index.js b/node_modules/kafkajs/src/protocol/recordBatch/record/v0/index.js new file mode 100644 index 0000000..2d0a8b3 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/recordBatch/record/v0/index.js @@ -0,0 +1,67 @@ +const Encoder = require('../../../encoder') +const Header = require('../../header/v0') + +/** + * v0 + * Record => + * Length => Varint + * Attributes => Int8 + * TimestampDelta => Varlong + * OffsetDelta => Varint + * Key => varInt|Bytes + * Value => varInt|Bytes + * Headers => [HeaderKey HeaderValue] + * HeaderKey => VarInt|String + * HeaderValue => VarInt|Bytes + */ + +/** + * @param [offsetDelta=0] {Integer} + * @param [timestampDelta=0] {Long} + * @param key {Buffer} + * @param value {Buffer} + * @param [headers={}] {Object} + */ +module.exports = ({ offsetDelta = 0, timestampDelta = 0, key, value, headers = {} }) => { + const headersArray = Object.keys(headers).flatMap(headerKey => + !Array.isArray(headers[headerKey]) + ? [{ key: headerKey, value: headers[headerKey] }] + : headers[headerKey].map(headerValue => ({ key: headerKey, value: headerValue })) + ) + + const sizeOfBody = + 1 + // always one byte for attributes + Encoder.sizeOfVarLong(timestampDelta) + + Encoder.sizeOfVarInt(offsetDelta) + + Encoder.sizeOfVarIntBytes(key) + + Encoder.sizeOfVarIntBytes(value) + + sizeOfHeaders(headersArray) + + return new Encoder() + .writeVarInt(sizeOfBody) + .writeInt8(0) // no used record attributes at the moment + .writeVarLong(timestampDelta) + .writeVarInt(offsetDelta) + .writeVarIntBytes(key) + .writeVarIntBytes(value) + .writeVarIntArray(headersArray.map(Header)) +} + +const sizeOfHeaders = headersArray => { + let size = Encoder.sizeOfVarInt(headersArray.length) + + for (const header of headersArray) { + const keySize = Buffer.byteLength(header.key) + const valueSize = Buffer.byteLength(header.value) + + size += Encoder.sizeOfVarInt(keySize) + keySize + + if (header.value === null) { + size += Encoder.sizeOfVarInt(-1) + } else { + size += Encoder.sizeOfVarInt(valueSize) + valueSize + } + } + + return size +} diff --git a/node_modules/kafkajs/src/protocol/recordBatch/v0/decoder.js b/node_modules/kafkajs/src/protocol/recordBatch/v0/decoder.js new file mode 100644 index 0000000..1d3e204 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/recordBatch/v0/decoder.js @@ -0,0 +1,118 @@ +const Decoder = require('../../decoder') +const { KafkaJSPartialMessageError } = require('../../../errors') +const { lookupCodecByAttributes } = require('../../message/compression') +const RecordDecoder = require('../record/v0/decoder') +const TimestampTypes = require('../../timestampTypes') + +const TIMESTAMP_TYPE_FLAG_MASK = 0x8 +const TRANSACTIONAL_FLAG_MASK = 0x10 +const CONTROL_FLAG_MASK = 0x20 + +/** + * v0 + * RecordBatch => + * FirstOffset => int64 + * Length => int32 + * PartitionLeaderEpoch => int32 + * Magic => int8 + * CRC => int32 + * Attributes => int16 + * LastOffsetDelta => int32 + * FirstTimestamp => int64 + * MaxTimestamp => int64 + * ProducerId => int64 + * ProducerEpoch => int16 + * FirstSequence => int32 + * Records => [Record] + */ + +module.exports = async fetchDecoder => { + const firstOffset = fetchDecoder.readInt64().toString() + const length = fetchDecoder.readInt32() + const decoder = fetchDecoder.slice(length) + fetchDecoder.forward(length) + + const remainingBytes = Buffer.byteLength(decoder.buffer) + + if (remainingBytes < length) { + throw new KafkaJSPartialMessageError( + `Tried to decode a partial record batch: remainingBytes(${remainingBytes}) < recordBatchLength(${length})` + ) + } + + const partitionLeaderEpoch = decoder.readInt32() + + // The magic byte was read by the Fetch protocol to distinguish between + // the record batch and the legacy message set. It's not used here but + // it has to be read. + const magicByte = decoder.readInt8() // eslint-disable-line no-unused-vars + + // The library is currently not performing CRC validations + const crc = decoder.readInt32() // eslint-disable-line no-unused-vars + + const attributes = decoder.readInt16() + const lastOffsetDelta = decoder.readInt32() + const firstTimestamp = decoder.readInt64().toString() + const maxTimestamp = decoder.readInt64().toString() + const producerId = decoder.readInt64().toString() + const producerEpoch = decoder.readInt16() + const firstSequence = decoder.readInt32() + + const inTransaction = (attributes & TRANSACTIONAL_FLAG_MASK) > 0 + const isControlBatch = (attributes & CONTROL_FLAG_MASK) > 0 + const timestampType = + (attributes & TIMESTAMP_TYPE_FLAG_MASK) > 0 + ? TimestampTypes.LOG_APPEND_TIME + : TimestampTypes.CREATE_TIME + + const codec = lookupCodecByAttributes(attributes) + + const recordContext = { + firstOffset, + firstTimestamp, + partitionLeaderEpoch, + inTransaction, + isControlBatch, + lastOffsetDelta, + producerId, + producerEpoch, + firstSequence, + maxTimestamp, + timestampType, + } + + const records = await decodeRecords(codec, decoder, { ...recordContext, magicByte }) + + return { + ...recordContext, + records, + } +} + +const decodeRecords = async (codec, recordsDecoder, recordContext) => { + if (!codec) { + return recordsDecoder.readArray(decoder => decodeRecord(decoder, recordContext)) + } + + const length = recordsDecoder.readInt32() + + if (length <= 0) { + return [] + } + + const compressedRecordsBuffer = recordsDecoder.readAll() + const decompressedRecordBuffer = await codec.decompress(compressedRecordsBuffer) + const decompressedRecordDecoder = new Decoder(decompressedRecordBuffer) + const records = new Array(length) + + for (let i = 0; i < length; i++) { + records[i] = decodeRecord(decompressedRecordDecoder, recordContext) + } + + return records +} + +const decodeRecord = (decoder, recordContext) => { + const recordBuffer = decoder.readVarIntBytes() + return RecordDecoder(new Decoder(recordBuffer), recordContext) +} diff --git a/node_modules/kafkajs/src/protocol/recordBatch/v0/index.js b/node_modules/kafkajs/src/protocol/recordBatch/v0/index.js new file mode 100644 index 0000000..599a940 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/recordBatch/v0/index.js @@ -0,0 +1,93 @@ +const Long = require('../../../utils/long') +const Encoder = require('../../encoder') +const crc32C = require('../crc32C') +const { + Types: Compression, + lookupCodec, + COMPRESSION_CODEC_MASK, +} = require('../../message/compression') + +const MAGIC_BYTE = 2 +const TIMESTAMP_MASK = 0 // The fourth lowest bit, always set this bit to 0 (since 0.10.0) +const TRANSACTIONAL_MASK = 16 // The fifth lowest bit + +/** + * v0 + * RecordBatch => + * FirstOffset => int64 + * Length => int32 + * PartitionLeaderEpoch => int32 + * Magic => int8 + * CRC => int32 + * Attributes => int16 + * LastOffsetDelta => int32 + * FirstTimestamp => int64 + * MaxTimestamp => int64 + * ProducerId => int64 + * ProducerEpoch => int16 + * FirstSequence => int32 + * Records => [Record] + */ + +const RecordBatch = async ({ + compression = Compression.None, + firstOffset = Long.fromInt(0), + firstTimestamp = Date.now(), + maxTimestamp = Date.now(), + partitionLeaderEpoch = 0, + lastOffsetDelta = 0, + transactional = false, + producerId = Long.fromValue(-1), // for idempotent messages + producerEpoch = 0, // for idempotent messages + firstSequence = 0, // for idempotent messages + records = [], +}) => { + const COMPRESSION_CODEC = compression & COMPRESSION_CODEC_MASK + const IN_TRANSACTION = transactional ? TRANSACTIONAL_MASK : 0 + const attributes = COMPRESSION_CODEC | TIMESTAMP_MASK | IN_TRANSACTION + + const batchBody = new Encoder() + .writeInt16(attributes) + .writeInt32(lastOffsetDelta) + .writeInt64(firstTimestamp) + .writeInt64(maxTimestamp) + .writeInt64(producerId) + .writeInt16(producerEpoch) + .writeInt32(firstSequence) + + if (compression === Compression.None) { + if (records.every(v => typeof v === typeof records[0])) { + batchBody.writeArray(records, typeof records[0]) + } else { + batchBody.writeArray(records) + } + } else { + const compressedRecords = await compressRecords(compression, records) + batchBody.writeInt32(records.length).writeBuffer(compressedRecords) + } + + // CRC32C validation is happening here: + // https://github.com/apache/kafka/blob/0.11.0.1/clients/src/main/java/org/apache/kafka/common/record/DefaultRecordBatch.java#L148 + + const batch = new Encoder() + .writeInt32(partitionLeaderEpoch) + .writeInt8(MAGIC_BYTE) + .writeUInt32(crc32C(batchBody.buffer)) + .writeEncoder(batchBody) + + return new Encoder().writeInt64(firstOffset).writeBytes(batch.buffer) +} + +const compressRecords = async (compression, records) => { + const codec = lookupCodec(compression) + const recordsEncoder = new Encoder() + + recordsEncoder.writeEncoderArray(records) + + return codec.compress(recordsEncoder) +} + +module.exports = { + RecordBatch, + MAGIC_BYTE, +} diff --git a/node_modules/kafkajs/src/protocol/request.js b/node_modules/kafkajs/src/protocol/request.js new file mode 100644 index 0000000..6881ca3 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/request.js @@ -0,0 +1,13 @@ +const Encoder = require('./encoder') + +module.exports = async ({ correlationId, clientId, request: { apiKey, apiVersion, encode } }) => { + const payload = await encode() + const requestPayload = new Encoder() + .writeInt16(apiKey) + .writeInt16(apiVersion) + .writeInt32(correlationId) + .writeString(clientId) + .writeEncoder(payload) + + return new Encoder().writeInt32(requestPayload.size()).writeEncoder(requestPayload) +} diff --git a/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/index.js b/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/index.js new file mode 100644 index 0000000..51a5dea --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/index.js @@ -0,0 +1,17 @@ +const versions = { + 0: ({ transactionalId, producerId, producerEpoch, groupId }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ transactionalId, producerId, producerEpoch, groupId }), response } + }, + 1: ({ transactionalId, producerId, producerEpoch, groupId }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ transactionalId, producerId, producerEpoch, groupId }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v0/request.js b/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v0/request.js new file mode 100644 index 0000000..f1069d2 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v0/request.js @@ -0,0 +1,23 @@ +const Encoder = require('../../../encoder') +const { AddOffsetsToTxn: apiKey } = require('../../apiKeys') + +/** + * AddOffsetsToTxn Request (Version: 0) => transactional_id producer_id producer_epoch group_id + * transactional_id => STRING + * producer_id => INT64 + * producer_epoch => INT16 + * group_id => STRING + */ + +module.exports = ({ transactionalId, producerId, producerEpoch, groupId }) => ({ + apiKey, + apiVersion: 0, + apiName: 'AddOffsetsToTxn', + encode: async () => { + return new Encoder() + .writeString(transactionalId) + .writeInt64(producerId) + .writeInt16(producerEpoch) + .writeString(groupId) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v0/response.js b/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v0/response.js new file mode 100644 index 0000000..ee992b3 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v0/response.js @@ -0,0 +1,33 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode, failIfVersionNotSupported } = require('../../../error') + +/** + * AddOffsetsToTxn Response (Version: 0) => throttle_time_ms error_code + * throttle_time_ms => INT32 + * error_code => INT16 + */ +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { + throttleTime, + errorCode, + } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v1/request.js b/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v1/request.js new file mode 100644 index 0000000..bba1586 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v1/request.js @@ -0,0 +1,20 @@ +const requestV0 = require('../v0/request') + +/** + * AddOffsetsToTxn Request (Version: 1) => transactional_id producer_id producer_epoch group_id + * transactional_id => STRING + * producer_id => INT64 + * producer_epoch => INT16 + * group_id => STRING + */ + +module.exports = ({ transactionalId, producerId, producerEpoch, groupId }) => + Object.assign( + requestV0({ + transactionalId, + producerId, + producerEpoch, + groupId, + }), + { apiVersion: 1 } + ) diff --git a/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v1/response.js b/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v1/response.js new file mode 100644 index 0000000..8adc33c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/addOffsetsToTxn/v1/response.js @@ -0,0 +1,24 @@ +const { parse, decode: decodeV0 } = require('../v0/response') + +/** + * Starting in version 1, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * AddOffsetsToTxn Response (Version: 1) => throttle_time_ms error_code + * throttle_time_ms => INT32 + * error_code => INT16 + */ +const decode = async rawData => { + const decoded = await decodeV0(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/index.js b/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/index.js new file mode 100644 index 0000000..ecffb4e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/index.js @@ -0,0 +1,17 @@ +const versions = { + 0: ({ transactionalId, producerId, producerEpoch, topics }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ transactionalId, producerId, producerEpoch, topics }), response } + }, + 1: ({ transactionalId, producerId, producerEpoch, topics }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ transactionalId, producerId, producerEpoch, topics }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v0/request.js b/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v0/request.js new file mode 100644 index 0000000..8215786 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v0/request.js @@ -0,0 +1,33 @@ +const Encoder = require('../../../encoder') +const { AddPartitionsToTxn: apiKey } = require('../../apiKeys') + +/** + * AddPartitionsToTxn Request (Version: 0) => transactional_id producer_id producer_epoch [topics] + * transactional_id => STRING + * producer_id => INT64 + * producer_epoch => INT16 + * topics => topic [partitions] + * topic => STRING + * partitions => INT32 + */ + +module.exports = ({ transactionalId, producerId, producerEpoch, topics }) => ({ + apiKey, + apiVersion: 0, + apiName: 'AddPartitionsToTxn', + encode: async () => { + return new Encoder() + .writeString(transactionalId) + .writeInt64(producerId) + .writeInt16(producerEpoch) + .writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = partition => { + return new Encoder().writeInt32(partition) +} diff --git a/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v0/response.js b/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v0/response.js new file mode 100644 index 0000000..817a365 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v0/response.js @@ -0,0 +1,51 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * AddPartitionsToTxn Response (Version: 0) => throttle_time_ms [errors] + * throttle_time_ms => INT32 + * errors => topic [partition_errors] + * topic => STRING + * partition_errors => partition error_code + * partition => INT32 + * error_code => INT16 + */ +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errors = await decoder.readArrayAsync(decodeError) + + return { + throttleTime, + errors, + } +} + +const decodeError = async decoder => ({ + topic: decoder.readString(), + partitionErrors: await decoder.readArrayAsync(decodePartitionError), +}) + +const decodePartitionError = decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), +}) + +const parse = async data => { + const topicsWithErrors = data.errors + .map(({ partitionErrors }) => ({ + partitionsWithErrors: partitionErrors.filter(({ errorCode }) => failure(errorCode)), + })) + .filter(({ partitionsWithErrors }) => partitionsWithErrors.length) + + if (topicsWithErrors.length > 0) { + throw createErrorFromCode(topicsWithErrors[0].partitionsWithErrors[0].errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v1/request.js b/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v1/request.js new file mode 100644 index 0000000..a618516 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v1/request.js @@ -0,0 +1,22 @@ +const requestV0 = require('../v0/request') + +/** + * AddPartitionsToTxn Request (Version: 1) => transactional_id producer_id producer_epoch [topics] + * transactional_id => STRING + * producer_id => INT64 + * producer_epoch => INT16 + * topics => topic [partitions] + * topic => STRING + * partitions => INT32 + */ + +module.exports = ({ transactionalId, producerId, producerEpoch, topics }) => + Object.assign( + requestV0({ + transactionalId, + producerId, + producerEpoch, + topics, + }), + { apiVersion: 1 } + ) diff --git a/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v1/response.js b/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v1/response.js new file mode 100644 index 0000000..fe64326 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/addPartitionsToTxn/v1/response.js @@ -0,0 +1,28 @@ +const { parse, decode: decodeV0 } = require('../v0/response') + +/** + * Starting in version 1, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * AddPartitionsToTxn Response (Version: 1) => throttle_time_ms [errors] + * throttle_time_ms => INT32 + * errors => topic [partition_errors] + * topic => STRING + * partition_errors => partition error_code + * partition => INT32 + * error_code => INT16 + */ +const decode = async rawData => { + const decoded = await decodeV0(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/alterConfigs/index.js b/node_modules/kafkajs/src/protocol/requests/alterConfigs/index.js new file mode 100644 index 0000000..8d39439 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/alterConfigs/index.js @@ -0,0 +1,17 @@ +const versions = { + 0: ({ resources, validateOnly }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ resources, validateOnly }), response } + }, + 1: ({ resources, validateOnly }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ resources, validateOnly }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/alterConfigs/v0/request.js b/node_modules/kafkajs/src/protocol/requests/alterConfigs/v0/request.js new file mode 100644 index 0000000..5b0554d --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/alterConfigs/v0/request.js @@ -0,0 +1,37 @@ +const Encoder = require('../../../encoder') +const { AlterConfigs: apiKey } = require('../../apiKeys') + +/** + * AlterConfigs Request (Version: 0) => [resources] validate_only + * resources => resource_type resource_name [config_entries] + * resource_type => INT8 + * resource_name => STRING + * config_entries => config_name config_value + * config_name => STRING + * config_value => NULLABLE_STRING + * validate_only => BOOLEAN + */ + +/** + * @param {Array} resources An array of resources to change + * @param {boolean} [validateOnly=false] + */ +module.exports = ({ resources, validateOnly = false }) => ({ + apiKey, + apiVersion: 0, + apiName: 'AlterConfigs', + encode: async () => { + return new Encoder().writeArray(resources.map(encodeResource)).writeBoolean(validateOnly) + }, +}) + +const encodeResource = ({ type, name, configEntries }) => { + return new Encoder() + .writeInt8(type) + .writeString(name) + .writeArray(configEntries.map(encodeConfigEntries)) +} + +const encodeConfigEntries = ({ name, value }) => { + return new Encoder().writeString(name).writeString(value) +} diff --git a/node_modules/kafkajs/src/protocol/requests/alterConfigs/v0/response.js b/node_modules/kafkajs/src/protocol/requests/alterConfigs/v0/response.js new file mode 100644 index 0000000..cf31d2f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/alterConfigs/v0/response.js @@ -0,0 +1,44 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * AlterConfigs Response (Version: 0) => throttle_time_ms [resources] + * throttle_time_ms => INT32 + * resources => error_code error_message resource_type resource_name + * error_code => INT16 + * error_message => NULLABLE_STRING + * resource_type => INT8 + * resource_name => STRING + */ + +const decodeResources = decoder => ({ + errorCode: decoder.readInt16(), + errorMessage: decoder.readString(), + resourceType: decoder.readInt8(), + resourceName: decoder.readString(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const resources = decoder.readArray(decodeResources) + + return { + throttleTime, + resources, + } +} + +const parse = async data => { + const resourcesWithError = data.resources.filter(({ errorCode }) => failure(errorCode)) + if (resourcesWithError.length > 0) { + throw createErrorFromCode(resourcesWithError[0].errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/alterConfigs/v1/request.js b/node_modules/kafkajs/src/protocol/requests/alterConfigs/v1/request.js new file mode 100644 index 0000000..3c9f003 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/alterConfigs/v1/request.js @@ -0,0 +1,25 @@ +const requestV0 = require('../v0/request') + +/** + * AlterConfigs Request (Version: 1) => [resources] validate_only + * resources => resource_type resource_name [config_entries] + * resource_type => INT8 + * resource_name => STRING + * config_entries => config_name config_value + * config_name => STRING + * config_value => NULLABLE_STRING + * validate_only => BOOLEAN + */ + +/** + * @param {Array} resources An array of resources to change + * @param {boolean} [validateOnly=false] + */ +module.exports = ({ resources, validateOnly }) => + Object.assign( + requestV0({ + resources, + validateOnly, + }), + { apiVersion: 1 } + ) diff --git a/node_modules/kafkajs/src/protocol/requests/alterConfigs/v1/response.js b/node_modules/kafkajs/src/protocol/requests/alterConfigs/v1/response.js new file mode 100644 index 0000000..44cb297 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/alterConfigs/v1/response.js @@ -0,0 +1,29 @@ +const { parse, decode: decodeV0 } = require('../v0/response') + +/** + * Starting in version 1, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * AlterConfigs Response (Version: 1) => throttle_time_ms [resources] + * throttle_time_ms => INT32 + * resources => error_code error_message resource_type resource_name + * error_code => INT16 + * error_message => NULLABLE_STRING + * resource_type => INT8 + * resource_name => STRING + */ + +const decode = async rawData => { + const decoded = await decodeV0(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/alterPartitionReassignments/index.js b/node_modules/kafkajs/src/protocol/requests/alterPartitionReassignments/index.js new file mode 100644 index 0000000..40847d6 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/alterPartitionReassignments/index.js @@ -0,0 +1,12 @@ +const versions = { + 0: ({ topics, timeout }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ topics, timeout }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/alterPartitionReassignments/v0/request.js b/node_modules/kafkajs/src/protocol/requests/alterPartitionReassignments/v0/request.js new file mode 100644 index 0000000..819baf6 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/alterPartitionReassignments/v0/request.js @@ -0,0 +1,43 @@ +const Encoder = require('../../../encoder') +const { AlterPartitionReassignments: apiKey } = require('../../apiKeys') + +/** + * AlterPartitionReassignments Request (Version: 0) => timeout_ms [topics] TAG_BUFFER + * timeout_ms => INT32 + * topics => name [partitions] TAG_BUFFER + * name => COMPACT_STRING + * partitions => partition_index [replicas] TAG_BUFFER + * partition_index => INT32 + * replicas => INT32 + */ + +module.exports = ({ topics, timeout = 5000 }) => ({ + apiKey, + apiVersion: 0, + apiName: 'AlterPartitionReassignments', + encode: async () => { + return new Encoder() + .writeUVarIntBytes() + .writeInt32(timeout) + .writeUVarIntArray(topics.map(encodeTopics)) + .writeUVarIntBytes() + }, +}) + +const encodeTopics = ({ topic, partitionAssignment }) => { + return new Encoder() + .writeUVarIntString(topic) + .writeUVarIntArray(partitionAssignment.map(encodePartitionAssignment)) + .writeUVarIntBytes() +} + +const encodePartitionAssignment = ({ partition, replicas }) => { + return new Encoder() + .writeInt32(partition) + .writeUVarIntArray(replicas.map(encodeReplicas)) + .writeUVarIntBytes() +} + +const encodeReplicas = replica => { + return new Encoder().writeInt32(replica) +} diff --git a/node_modules/kafkajs/src/protocol/requests/alterPartitionReassignments/v0/response.js b/node_modules/kafkajs/src/protocol/requests/alterPartitionReassignments/v0/response.js new file mode 100644 index 0000000..5de50a3 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/alterPartitionReassignments/v0/response.js @@ -0,0 +1,88 @@ +const { + KafkaJSAggregateError, + KafkaJSAlterPartitionReassignmentsError, +} = require('../../../../errors') +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * AlterPartitionReassignments Response (Version: 0) => throttle_time_ms error_code error_message [responses] TAG_BUFFER + * throttle_time_ms => INT32 + * error_code => INT16 + * error_message => COMPACT_NULLABLE_STRING + * responses => name [partitions] TAG_BUFFER + * name => COMPACT_STRING + * partitions => partition_index error_code error_message TAG_BUFFER + * partition_index => INT32 + * error_code => INT16 + * error_message => COMPACT_NULLABLE_STRING + */ + +const decodeResponses = decoder => { + const response = { + topic: decoder.readUVarIntString(), + partitions: decoder.readUVarIntArray(decodePartitions), + } + + decoder.readTaggedFields() + return response +} + +const decodePartitions = decoder => { + const partition = { + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + } + decoder.readUVarIntString() + decoder.readTaggedFields() + return partition +} + +const decode = async rawData => { + const decoder = new Decoder(rawData) + decoder.readTaggedFields() + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + decoder.readUVarIntString() + return { + throttleTime, + errorCode, + responses: decoder.readUVarIntArray(decodeResponses), + } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw new KafkaJSAlterPartitionReassignmentsError(createErrorFromCode(data.errorCode)) + } + + const topicPartitionsWithError = data.responses.flatMap(({ partitions, topic }) => + partitions + .filter(partition => failure(partition.errorCode)) + .map(partition => ({ + ...partition, + topic, + })) + ) + + if (topicPartitionsWithError.length > 0) { + throw new KafkaJSAggregateError( + 'Errors altering partition reassignments', + topicPartitionsWithError.map( + ({ topic, partition, errorCode }) => + new KafkaJSAlterPartitionReassignmentsError( + createErrorFromCode(errorCode), + topic, + partition + ) + ) + ) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/apiKeys.js b/node_modules/kafkajs/src/protocol/requests/apiKeys.js new file mode 100644 index 0000000..442739e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/apiKeys.js @@ -0,0 +1,49 @@ +module.exports = { + Produce: 0, + Fetch: 1, + ListOffsets: 2, + Metadata: 3, + LeaderAndIsr: 4, + StopReplica: 5, + UpdateMetadata: 6, + ControlledShutdown: 7, + OffsetCommit: 8, + OffsetFetch: 9, + GroupCoordinator: 10, + JoinGroup: 11, + Heartbeat: 12, + LeaveGroup: 13, + SyncGroup: 14, + DescribeGroups: 15, + ListGroups: 16, + SaslHandshake: 17, + ApiVersions: 18, // ApiVersions v0 on Kafka 0.10 + CreateTopics: 19, + DeleteTopics: 20, + DeleteRecords: 21, + InitProducerId: 22, + OffsetForLeaderEpoch: 23, + AddPartitionsToTxn: 24, + AddOffsetsToTxn: 25, + EndTxn: 26, + WriteTxnMarkers: 27, + TxnOffsetCommit: 28, + DescribeAcls: 29, + CreateAcls: 30, + DeleteAcls: 31, + DescribeConfigs: 32, + AlterConfigs: 33, // ApiVersions v0 and v1 on Kafka 0.11 + AlterReplicaLogDirs: 34, + DescribeLogDirs: 35, + SaslAuthenticate: 36, + CreatePartitions: 37, + CreateDelegationToken: 38, + RenewDelegationToken: 39, + ExpireDelegationToken: 40, + DescribeDelegationToken: 41, + DeleteGroups: 42, // ApiVersions v2 on Kafka 1.0 + ElectPreferredLeaders: 43, + IncrementalAlterConfigs: 44, + AlterPartitionReassignments: 45, + ListPartitionReassignments: 46, +} diff --git a/node_modules/kafkajs/src/protocol/requests/apiVersions/index.js b/node_modules/kafkajs/src/protocol/requests/apiVersions/index.js new file mode 100644 index 0000000..90d3939 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/apiVersions/index.js @@ -0,0 +1,24 @@ +const logResponseError = false + +const versions = { + 0: () => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request(), response, logResponseError: true } + }, + 1: () => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request(), response, logResponseError } + }, + 2: () => { + const request = require('./v2/request') + const response = require('./v2/response') + return { request: request(), response, logResponseError } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/apiVersions/v0/request.js b/node_modules/kafkajs/src/protocol/requests/apiVersions/v0/request.js new file mode 100644 index 0000000..fa9ed46 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/apiVersions/v0/request.js @@ -0,0 +1,13 @@ +const Encoder = require('../../../encoder') +const { ApiVersions: apiKey } = require('../../apiKeys') + +/** + * ApiVersionRequest => ApiKeys + */ + +module.exports = () => ({ + apiKey, + apiVersion: 0, + apiName: 'ApiVersions', + encode: async () => new Encoder(), +}) diff --git a/node_modules/kafkajs/src/protocol/requests/apiVersions/v0/response.js b/node_modules/kafkajs/src/protocol/requests/apiVersions/v0/response.js new file mode 100644 index 0000000..beb8d68 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/apiVersions/v0/response.js @@ -0,0 +1,43 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode, failIfVersionNotSupported } = require('../../../error') + +/** + * ApiVersionResponse => ApiVersions + * ErrorCode = INT16 + * ApiVersions = [ApiVersion] + * ApiVersion = ApiKey MinVersion MaxVersion + * ApiKey = INT16 + * MinVersion = INT16 + * MaxVersion = INT16 + */ + +const apiVersion = decoder => ({ + apiKey: decoder.readInt16(), + minVersion: decoder.readInt16(), + maxVersion: decoder.readInt16(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { + errorCode, + apiVersions: decoder.readArray(apiVersion), + } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/apiVersions/v1/request.js b/node_modules/kafkajs/src/protocol/requests/apiVersions/v1/request.js new file mode 100644 index 0000000..ebf44d6 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/apiVersions/v1/request.js @@ -0,0 +1,5 @@ +const requestV0 = require('../v0/request') + +// ApiVersions Request after v1 indicates the client can parse throttle_time_ms + +module.exports = () => ({ ...requestV0(), apiVersion: 1 }) diff --git a/node_modules/kafkajs/src/protocol/requests/apiVersions/v1/response.js b/node_modules/kafkajs/src/protocol/requests/apiVersions/v1/response.js new file mode 100644 index 0000000..149c240 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/apiVersions/v1/response.js @@ -0,0 +1,49 @@ +const Decoder = require('../../../decoder') +const { failIfVersionNotSupported } = require('../../../error') +const { parse: parseV0 } = require('../v0/response') + +/** + * ApiVersions Response (Version: 1) => error_code [api_versions] throttle_time_ms + * error_code => INT16 + * api_versions => api_key min_version max_version + * api_key => INT16 + * min_version => INT16 + * max_version => INT16 + * throttle_time_ms => INT32 + */ + +const apiVersion = decoder => ({ + apiKey: decoder.readInt16(), + minVersion: decoder.readInt16(), + maxVersion: decoder.readInt16(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + const apiVersions = decoder.readArray(apiVersion) + + /** + * The Java client defaults this value to 0 if not present, + * even though it is required in the protocol. This is to + * work around https://github.com/tulios/kafkajs/issues/491 + * + * See: + * https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/protocol/CommonFields.java#L23-L25 + */ + const throttleTime = decoder.canReadInt32() ? decoder.readInt32() : 0 + + return { + errorCode, + apiVersions, + throttleTime, + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/apiVersions/v2/request.js b/node_modules/kafkajs/src/protocol/requests/apiVersions/v2/request.js new file mode 100644 index 0000000..88785eb --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/apiVersions/v2/request.js @@ -0,0 +1,5 @@ +const requestV0 = require('../v0/request') + +// ApiVersions Request after v1 indicates the client can parse throttle_time_ms + +module.exports = () => ({ ...requestV0(), apiVersion: 2 }) diff --git a/node_modules/kafkajs/src/protocol/requests/apiVersions/v2/response.js b/node_modules/kafkajs/src/protocol/requests/apiVersions/v2/response.js new file mode 100644 index 0000000..2a343a1 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/apiVersions/v2/response.js @@ -0,0 +1,29 @@ +const { parse, decode: decodeV1 } = require('../v1/response') + +/** + * Starting in version 2, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * ApiVersions Response (Version: 2) => error_code [api_versions] throttle_time_ms + * error_code => INT16 + * api_versions => api_key min_version max_version + * api_key => INT16 + * min_version => INT16 + * max_version => INT16 + * throttle_time_ms => INT32 + */ + +const decode = async rawData => { + const decoded = await decodeV1(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/createAcls/index.js b/node_modules/kafkajs/src/protocol/requests/createAcls/index.js new file mode 100644 index 0000000..6cc4f40 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createAcls/index.js @@ -0,0 +1,17 @@ +const versions = { + 0: ({ creations }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ creations }), response } + }, + 1: ({ creations }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ creations }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/createAcls/v0/request.js b/node_modules/kafkajs/src/protocol/requests/createAcls/v0/request.js new file mode 100644 index 0000000..4a2b02f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createAcls/v0/request.js @@ -0,0 +1,39 @@ +const Encoder = require('../../../encoder') +const { CreateAcls: apiKey } = require('../../apiKeys') + +/** + * CreateAcls Request (Version: 0) => [creations] + * creations => resource_type resource_name principal host operation permission_type + * resource_type => INT8 + * resource_name => STRING + * principal => STRING + * host => STRING + * operation => INT8 + * permission_type => INT8 + */ + +const encodeCreations = ({ + resourceType, + resourceName, + principal, + host, + operation, + permissionType, +}) => { + return new Encoder() + .writeInt8(resourceType) + .writeString(resourceName) + .writeString(principal) + .writeString(host) + .writeInt8(operation) + .writeInt8(permissionType) +} + +module.exports = ({ creations }) => ({ + apiKey, + apiVersion: 0, + apiName: 'CreateAcls', + encode: async () => { + return new Encoder().writeArray(creations.map(encodeCreations)) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/createAcls/v0/response.js b/node_modules/kafkajs/src/protocol/requests/createAcls/v0/response.js new file mode 100644 index 0000000..32a56fc --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createAcls/v0/response.js @@ -0,0 +1,43 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * CreateAcls Response (Version: 0) => throttle_time_ms [creation_responses] + * throttle_time_ms => INT32 + * creation_responses => error_code error_message + * error_code => INT16 + * error_message => NULLABLE_STRING + */ + +const decodeCreationResponse = decoder => ({ + errorCode: decoder.readInt16(), + errorMessage: decoder.readString(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const creationResponses = decoder.readArray(decodeCreationResponse) + + return { + throttleTime, + creationResponses, + } +} + +const parse = async data => { + const creationResponsesWithError = data.creationResponses.filter(({ errorCode }) => + failure(errorCode) + ) + + if (creationResponsesWithError.length > 0) { + throw createErrorFromCode(creationResponsesWithError[0].errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/createAcls/v1/request.js b/node_modules/kafkajs/src/protocol/requests/createAcls/v1/request.js new file mode 100644 index 0000000..d1000f9 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createAcls/v1/request.js @@ -0,0 +1,42 @@ +const Encoder = require('../../../encoder') +const { CreateAcls: apiKey } = require('../../apiKeys') + +/** + * CreateAcls Request (Version: 1) => [creations] + * creations => resource_type resource_name resource_pattern_type principal host operation permission_type + * resource_type => INT8 + * resource_name => STRING + * resource_pattern_type => INT8 + * principal => STRING + * host => STRING + * operation => INT8 + * permission_type => INT8 + */ + +const encodeCreations = ({ + resourceType, + resourceName, + resourcePatternType, + principal, + host, + operation, + permissionType, +}) => { + return new Encoder() + .writeInt8(resourceType) + .writeString(resourceName) + .writeInt8(resourcePatternType) + .writeString(principal) + .writeString(host) + .writeInt8(operation) + .writeInt8(permissionType) +} + +module.exports = ({ creations }) => ({ + apiKey, + apiVersion: 1, + apiName: 'CreateAcls', + encode: async () => { + return new Encoder().writeArray(creations.map(encodeCreations)) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/createAcls/v1/response.js b/node_modules/kafkajs/src/protocol/requests/createAcls/v1/response.js new file mode 100644 index 0000000..a736343 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createAcls/v1/response.js @@ -0,0 +1,27 @@ +const { parse, decode: decodeV0 } = require('../v0/response') + +/** + * Starting in version 1, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * CreateAcls Response (Version: 1) => throttle_time_ms [creation_responses] + * throttle_time_ms => INT32 + * creation_responses => error_code error_message + * error_code => INT16 + * error_message => NULLABLE_STRING + */ + +const decode = async rawData => { + const decoded = await decodeV0(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/createPartitions/index.js b/node_modules/kafkajs/src/protocol/requests/createPartitions/index.js new file mode 100644 index 0000000..eed8429 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createPartitions/index.js @@ -0,0 +1,17 @@ +const versions = { + 0: ({ topicPartitions, timeout, validateOnly }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ topicPartitions, timeout, validateOnly }), response } + }, + 1: ({ topicPartitions, validateOnly, timeout }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ topicPartitions, validateOnly, timeout }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/createPartitions/v0/request.js b/node_modules/kafkajs/src/protocol/requests/createPartitions/v0/request.js new file mode 100644 index 0000000..bff9ab0 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createPartitions/v0/request.js @@ -0,0 +1,36 @@ +const Encoder = require('../../../encoder') +const { CreatePartitions: apiKey } = require('../../apiKeys') + +/** + * CreatePartitions Request (Version: 0) => [topic_partitions] timeout validate_only + * topic_partitions => topic new_partitions + * topic => STRING + * new_partitions => count [assignment] + * count => INT32 + * assignment => ARRAY(INT32) + * timeout => INT32 + * validate_only => BOOLEAN + */ + +module.exports = ({ topicPartitions, validateOnly = false, timeout = 5000 }) => ({ + apiKey, + apiVersion: 0, + apiName: 'CreatePartitions', + encode: async () => { + return new Encoder() + .writeArray(topicPartitions.map(encodeTopicPartitions)) + .writeInt32(timeout) + .writeBoolean(validateOnly) + }, +}) + +const encodeTopicPartitions = ({ topic, count, assignments = [] }) => { + return new Encoder() + .writeString(topic) + .writeInt32(count) + .writeNullableArray(assignments.map(encodeAssignments)) +} + +const encodeAssignments = brokerIds => { + return new Encoder().writeNullableArray(brokerIds) +} diff --git a/node_modules/kafkajs/src/protocol/requests/createPartitions/v0/response.js b/node_modules/kafkajs/src/protocol/requests/createPartitions/v0/response.js new file mode 100644 index 0000000..583b8ad --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createPartitions/v0/response.js @@ -0,0 +1,42 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/* + * CreatePartitions Response (Version: 0) => throttle_time_ms [topic_errors] + * throttle_time_ms => INT32 + * topic_errors => topic error_code error_message + * topic => STRING + * error_code => INT16 + * error_message => NULLABLE_STRING + */ + +const topicNameComparator = (a, b) => a.topic.localeCompare(b.topic) + +const topicErrors = decoder => ({ + topic: decoder.readString(), + errorCode: decoder.readInt16(), + errorMessage: decoder.readString(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + return { + throttleTime, + topicErrors: decoder.readArray(topicErrors).sort(topicNameComparator), + } +} + +const parse = async data => { + const topicsWithError = data.topicErrors.filter(({ errorCode }) => failure(errorCode)) + if (topicsWithError.length > 0) { + throw createErrorFromCode(topicsWithError[0].errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/createPartitions/v1/request.js b/node_modules/kafkajs/src/protocol/requests/createPartitions/v1/request.js new file mode 100644 index 0000000..df71594 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createPartitions/v1/request.js @@ -0,0 +1,15 @@ +const requestV0 = require('../v0/request') + +/** + * CreatePartitions Request (Version: 1) => [topic_partitions] timeout validate_only + * topic_partitions => topic new_partitions + * topic => STRING + * new_partitions => count [assignment] + * count => INT32 + * assignment => ARRAY(INT32) + * timeout => INT32 + * validate_only => BOOLEAN + */ + +module.exports = ({ topicPartitions, validateOnly, timeout }) => + Object.assign(requestV0({ topicPartitions, validateOnly, timeout }), { apiVersion: 1 }) diff --git a/node_modules/kafkajs/src/protocol/requests/createPartitions/v1/response.js b/node_modules/kafkajs/src/protocol/requests/createPartitions/v1/response.js new file mode 100644 index 0000000..37df8dc --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createPartitions/v1/response.js @@ -0,0 +1,28 @@ +const { parse, decode: decodeV0 } = require('../v0/response') + +/** + * Starting in version 1, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * CreatePartitions Response (Version: 0) => throttle_time_ms [topic_errors] + * throttle_time_ms => INT32 + * topic_errors => topic error_code error_message + * topic => STRING + * error_code => INT16 + * error_message => NULLABLE_STRING + */ + +const decode = async rawData => { + const decoded = await decodeV0(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/createTopics/index.js b/node_modules/kafkajs/src/protocol/requests/createTopics/index.js new file mode 100644 index 0000000..6cea340 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createTopics/index.js @@ -0,0 +1,27 @@ +const versions = { + 0: ({ topics, timeout }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ topics, timeout }), response } + }, + 1: ({ topics, validateOnly, timeout }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ topics, validateOnly, timeout }), response } + }, + 2: ({ topics, validateOnly, timeout }) => { + const request = require('./v2/request') + const response = require('./v2/response') + return { request: request({ topics, validateOnly, timeout }), response } + }, + 3: ({ topics, validateOnly, timeout }) => { + const request = require('./v3/request') + const response = require('./v3/response') + return { request: request({ topics, validateOnly, timeout }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/createTopics/v0/request.js b/node_modules/kafkajs/src/protocol/requests/createTopics/v0/request.js new file mode 100644 index 0000000..2a635d8 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createTopics/v0/request.js @@ -0,0 +1,49 @@ +const Encoder = require('../../../encoder') +const { CreateTopics: apiKey } = require('../../apiKeys') + +/** + * CreateTopics Request (Version: 0) => [create_topic_requests] timeout + * create_topic_requests => topic num_partitions replication_factor [replica_assignment] [config_entries] + * topic => STRING + * num_partitions => INT32 + * replication_factor => INT16 + * replica_assignment => partition [replicas] + * partition => INT32 + * replicas => INT32 + * config_entries => config_name config_value + * config_name => STRING + * config_value => NULLABLE_STRING + * timeout => INT32 + */ + +module.exports = ({ topics, timeout = 5000 }) => ({ + apiKey, + apiVersion: 0, + apiName: 'CreateTopics', + encode: async () => { + return new Encoder().writeArray(topics.map(encodeTopics)).writeInt32(timeout) + }, +}) + +const encodeTopics = ({ + topic, + numPartitions = -1, + replicationFactor = -1, + replicaAssignment = [], + configEntries = [], +}) => { + return new Encoder() + .writeString(topic) + .writeInt32(numPartitions) + .writeInt16(replicationFactor) + .writeArray(replicaAssignment.map(encodeReplicaAssignment)) + .writeArray(configEntries.map(encodeConfigEntries)) +} + +const encodeReplicaAssignment = ({ partition, replicas }) => { + return new Encoder().writeInt32(partition).writeArray(replicas) +} + +const encodeConfigEntries = ({ name, value }) => { + return new Encoder().writeString(name).writeString(value) +} diff --git a/node_modules/kafkajs/src/protocol/requests/createTopics/v0/response.js b/node_modules/kafkajs/src/protocol/requests/createTopics/v0/response.js new file mode 100644 index 0000000..d5642f9 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createTopics/v0/response.js @@ -0,0 +1,43 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') +const { KafkaJSAggregateError, KafkaJSCreateTopicError } = require('../../../../errors') + +/** + * CreateTopics Response (Version: 0) => [topic_errors] + * topic_errors => topic error_code + * topic => STRING + * error_code => INT16 + */ + +const topicNameComparator = (a, b) => a.topic.localeCompare(b.topic) + +const topicErrors = decoder => ({ + topic: decoder.readString(), + errorCode: decoder.readInt16(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + topicErrors: decoder.readArray(topicErrors).sort(topicNameComparator), + } +} + +const parse = async data => { + const topicsWithError = data.topicErrors.filter(({ errorCode }) => failure(errorCode)) + if (topicsWithError.length > 0) { + throw new KafkaJSAggregateError( + 'Topic creation errors', + topicsWithError.map( + error => new KafkaJSCreateTopicError(createErrorFromCode(error.errorCode), error.topic) + ) + ) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/createTopics/v1/request.js b/node_modules/kafkajs/src/protocol/requests/createTopics/v1/request.js new file mode 100644 index 0000000..151a014 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createTopics/v1/request.js @@ -0,0 +1,53 @@ +const Encoder = require('../../../encoder') +const { CreateTopics: apiKey } = require('../../apiKeys') + +/** + *CreateTopics Request (Version: 1) => [create_topic_requests] timeout validate_only + * create_topic_requests => topic num_partitions replication_factor [replica_assignment] [config_entries] + * topic => STRING + * num_partitions => INT32 + * replication_factor => INT16 + * replica_assignment => partition [replicas] + * partition => INT32 + * replicas => INT32 + * config_entries => config_name config_value + * config_name => STRING + * config_value => NULLABLE_STRING + * timeout => INT32 + * validate_only => BOOLEAN + */ + +module.exports = ({ topics, validateOnly = false, timeout = 5000 }) => ({ + apiKey, + apiVersion: 1, + apiName: 'CreateTopics', + encode: async () => { + return new Encoder() + .writeArray(topics.map(encodeTopics)) + .writeInt32(timeout) + .writeBoolean(validateOnly) + }, +}) + +const encodeTopics = ({ + topic, + numPartitions = -1, + replicationFactor = -1, + replicaAssignment = [], + configEntries = [], +}) => { + return new Encoder() + .writeString(topic) + .writeInt32(numPartitions) + .writeInt16(replicationFactor) + .writeArray(replicaAssignment.map(encodeReplicaAssignment)) + .writeArray(configEntries.map(encodeConfigEntries)) +} + +const encodeReplicaAssignment = ({ partition, replicas }) => { + return new Encoder().writeInt32(partition).writeArray(replicas) +} + +const encodeConfigEntries = ({ name, value }) => { + return new Encoder().writeString(name).writeString(value) +} diff --git a/node_modules/kafkajs/src/protocol/requests/createTopics/v1/response.js b/node_modules/kafkajs/src/protocol/requests/createTopics/v1/response.js new file mode 100644 index 0000000..917f900 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createTopics/v1/response.js @@ -0,0 +1,30 @@ +const Decoder = require('../../../decoder') +const { parse: parseV0 } = require('../v0/response') + +/** + * CreateTopics Response (Version: 1) => [topic_errors] + * topic_errors => topic error_code error_message + * topic => STRING + * error_code => INT16 + * error_message => NULLABLE_STRING + */ + +const topicNameComparator = (a, b) => a.topic.localeCompare(b.topic) + +const topicErrors = decoder => ({ + topic: decoder.readString(), + errorCode: decoder.readInt16(), + errorMessage: decoder.readString(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + topicErrors: decoder.readArray(topicErrors).sort(topicNameComparator), + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/createTopics/v2/request.js b/node_modules/kafkajs/src/protocol/requests/createTopics/v2/request.js new file mode 100644 index 0000000..122141c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createTopics/v2/request.js @@ -0,0 +1,20 @@ +const requestV1 = require('../v1/request') + +/** + * CreateTopics Request (Version: 2) => [create_topic_requests] timeout validate_only + * create_topic_requests => topic num_partitions replication_factor [replica_assignment] [config_entries] + * topic => STRING + * num_partitions => INT32 + * replication_factor => INT16 + * replica_assignment => partition [replicas] + * partition => INT32 + * replicas => INT32 + * config_entries => config_name config_value + * config_name => STRING + * config_value => NULLABLE_STRING + * timeout => INT32 + * validate_only => BOOLEAN + */ + +module.exports = ({ topics, validateOnly, timeout }) => + Object.assign(requestV1({ topics, validateOnly, timeout }), { apiVersion: 2 }) diff --git a/node_modules/kafkajs/src/protocol/requests/createTopics/v2/response.js b/node_modules/kafkajs/src/protocol/requests/createTopics/v2/response.js new file mode 100644 index 0000000..53ee179 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createTopics/v2/response.js @@ -0,0 +1,32 @@ +const Decoder = require('../../../decoder') +const { parse: parseV1 } = require('../v1/response') + +/** + * CreateTopics Response (Version: 2) => throttle_time_ms [topic_errors] + * throttle_time_ms => INT32 + * topic_errors => topic error_code error_message + * topic => STRING + * error_code => INT16 + * error_message => NULLABLE_STRING + */ + +const topicNameComparator = (a, b) => a.topic.localeCompare(b.topic) + +const topicErrors = decoder => ({ + topic: decoder.readString(), + errorCode: decoder.readInt16(), + errorMessage: decoder.readString(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + throttleTime: decoder.readInt32(), + topicErrors: decoder.readArray(topicErrors).sort(topicNameComparator), + } +} + +module.exports = { + decode, + parse: parseV1, +} diff --git a/node_modules/kafkajs/src/protocol/requests/createTopics/v3/request.js b/node_modules/kafkajs/src/protocol/requests/createTopics/v3/request.js new file mode 100644 index 0000000..9cbcfa8 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createTopics/v3/request.js @@ -0,0 +1,20 @@ +const requestV2 = require('../v2/request') + +/** + * CreateTopics Request (Version: 3) => [create_topic_requests] timeout validate_only + * create_topic_requests => topic num_partitions replication_factor [replica_assignment] [config_entries] + * topic => STRING + * num_partitions => INT32 + * replication_factor => INT16 + * replica_assignment => partition [replicas] + * partition => INT32 + * replicas => INT32 + * config_entries => config_name config_value + * config_name => STRING + * config_value => NULLABLE_STRING + * timeout => INT32 + * validate_only => BOOLEAN + */ + +module.exports = ({ topics, validateOnly, timeout }) => + Object.assign(requestV2({ topics, validateOnly, timeout }), { apiVersion: 3 }) diff --git a/node_modules/kafkajs/src/protocol/requests/createTopics/v3/response.js b/node_modules/kafkajs/src/protocol/requests/createTopics/v3/response.js new file mode 100644 index 0000000..3f857b7 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/createTopics/v3/response.js @@ -0,0 +1,28 @@ +const { parse, decode: decodeV2 } = require('../v2/response') + +/** + * Starting in version 3, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * CreateTopics Response (Version: 3) => throttle_time_ms [topic_errors] + * throttle_time_ms => INT32 + * topic_errors => topic error_code error_message + * topic => STRING + * error_code => INT16 + * error_message => NULLABLE_STRING + */ + +const decode = async rawData => { + const decoded = await decodeV2(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/deleteAcls/index.js b/node_modules/kafkajs/src/protocol/requests/deleteAcls/index.js new file mode 100644 index 0000000..81b3f5b --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteAcls/index.js @@ -0,0 +1,17 @@ +const versions = { + 0: ({ filters }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ filters }), response } + }, + 1: ({ filters }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ filters }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/deleteAcls/v0/request.js b/node_modules/kafkajs/src/protocol/requests/deleteAcls/v0/request.js new file mode 100644 index 0000000..59e1b27 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteAcls/v0/request.js @@ -0,0 +1,39 @@ +const Encoder = require('../../../encoder') +const { DeleteAcls: apiKey } = require('../../apiKeys') + +/** + * DeleteAcls Request (Version: 0) => [filters] + * filters => resource_type resource_name principal host operation permission_type + * resource_type => INT8 + * resource_name => NULLABLE_STRING + * principal => NULLABLE_STRING + * host => NULLABLE_STRING + * operation => INT8 + * permission_type => INT8 + */ + +const encodeFilters = ({ + resourceType, + resourceName, + principal, + host, + operation, + permissionType, +}) => { + return new Encoder() + .writeInt8(resourceType) + .writeString(resourceName) + .writeString(principal) + .writeString(host) + .writeInt8(operation) + .writeInt8(permissionType) +} + +module.exports = ({ filters }) => ({ + apiKey, + apiVersion: 0, + apiName: 'DeleteAcls', + encode: async () => { + return new Encoder().writeArray(filters.map(encodeFilters)) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/deleteAcls/v0/response.js b/node_modules/kafkajs/src/protocol/requests/deleteAcls/v0/response.js new file mode 100644 index 0000000..40ac23c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteAcls/v0/response.js @@ -0,0 +1,75 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * DeleteAcls Response (Version: 0) => throttle_time_ms [filter_responses] + * throttle_time_ms => INT32 + * filter_responses => error_code error_message [matching_acls] + * error_code => INT16 + * error_message => NULLABLE_STRING + * matching_acls => error_code error_message resource_type resource_name principal host operation permission_type + * error_code => INT16 + * error_message => NULLABLE_STRING + * resource_type => INT8 + * resource_name => STRING + * principal => STRING + * host => STRING + * operation => INT8 + * permission_type => INT8 + */ + +const decodeMatchingAcls = decoder => ({ + errorCode: decoder.readInt16(), + errorMessage: decoder.readString(), + resourceType: decoder.readInt8(), + resourceName: decoder.readString(), + principal: decoder.readString(), + host: decoder.readString(), + operation: decoder.readInt8(), + permissionType: decoder.readInt8(), +}) + +const decodeFilterResponse = decoder => ({ + errorCode: decoder.readInt16(), + errorMessage: decoder.readString(), + matchingAcls: decoder.readArray(decodeMatchingAcls), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const filterResponses = decoder.readArray(decodeFilterResponse) + + return { + throttleTime, + filterResponses, + } +} + +const parse = async data => { + const filterResponsesWithError = data.filterResponses.filter(({ errorCode }) => + failure(errorCode) + ) + + if (filterResponsesWithError.length > 0) { + throw createErrorFromCode(filterResponsesWithError[0].errorCode) + } + + for (const filterResponse of data.filterResponses) { + const matchingAcls = filterResponse.matchingAcls + const matchingAclsWithError = matchingAcls.filter(({ errorCode }) => failure(errorCode)) + + if (matchingAclsWithError.length > 0) { + throw createErrorFromCode(matchingAclsWithError[0].errorCode) + } + } + + return data +} + +module.exports = { + decodeMatchingAcls, + decodeFilterResponse, + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/deleteAcls/v1/request.js b/node_modules/kafkajs/src/protocol/requests/deleteAcls/v1/request.js new file mode 100644 index 0000000..5db3eb0 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteAcls/v1/request.js @@ -0,0 +1,42 @@ +const Encoder = require('../../../encoder') +const { DeleteAcls: apiKey } = require('../../apiKeys') + +/** + * DeleteAcls Request (Version: 1) => [filters] + * filters => resource_type resource_name resource_pattern_type_filter principal host operation permission_type + * resource_type => INT8 + * resource_name => NULLABLE_STRING + * resource_pattern_type_filter => INT8 + * principal => NULLABLE_STRING + * host => NULLABLE_STRING + * operation => INT8 + * permission_type => INT8 + */ + +const encodeFilters = ({ + resourceType, + resourceName, + resourcePatternType, + principal, + host, + operation, + permissionType, +}) => { + return new Encoder() + .writeInt8(resourceType) + .writeString(resourceName) + .writeInt8(resourcePatternType) + .writeString(principal) + .writeString(host) + .writeInt8(operation) + .writeInt8(permissionType) +} + +module.exports = ({ filters }) => ({ + apiKey, + apiVersion: 1, + apiName: 'DeleteAcls', + encode: async () => { + return new Encoder().writeArray(filters.map(encodeFilters)) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/deleteAcls/v1/response.js b/node_modules/kafkajs/src/protocol/requests/deleteAcls/v1/response.js new file mode 100644 index 0000000..97f6e11 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteAcls/v1/response.js @@ -0,0 +1,60 @@ +const Decoder = require('../../../decoder') +const { parse: parseV0 } = require('../v0/response') + +/** + * Starting in version 1, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * Version 1 also introduces a new resource pattern type field. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-290%3A+Support+for+Prefixed+ACLs + * + * DeleteAcls Response (Version: 1) => throttle_time_ms [filter_responses] + * throttle_time_ms => INT32 + * filter_responses => error_code error_message [matching_acls] + * error_code => INT16 + * error_message => NULLABLE_STRING + * matching_acls => error_code error_message resource_type resource_name resource_pattern_type principal host operation permission_type + * error_code => INT16 + * error_message => NULLABLE_STRING + * resource_type => INT8 + * resource_name => STRING + * resource_pattern_type => INT8 + * principal => STRING + * host => STRING + * operation => INT8 + * permission_type => INT8 + */ + +const decodeMatchingAcls = decoder => ({ + errorCode: decoder.readInt16(), + errorMessage: decoder.readString(), + resourceType: decoder.readInt8(), + resourceName: decoder.readString(), + resourcePatternType: decoder.readInt8(), + principal: decoder.readString(), + host: decoder.readString(), + operation: decoder.readInt8(), + permissionType: decoder.readInt8(), +}) + +const decodeFilterResponse = decoder => ({ + errorCode: decoder.readInt16(), + errorMessage: decoder.readString(), + matchingAcls: decoder.readArray(decodeMatchingAcls), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const filterResponses = decoder.readArray(decodeFilterResponse) + + return { + throttleTime: 0, + clientSideThrottleTime: throttleTime, + filterResponses, + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/deleteGroups/index.js b/node_modules/kafkajs/src/protocol/requests/deleteGroups/index.js new file mode 100644 index 0000000..32fda47 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteGroups/index.js @@ -0,0 +1,17 @@ +const versions = { + 0: groupIds => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request(groupIds), response } + }, + 1: groupIds => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request(groupIds), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/deleteGroups/v0/request.js b/node_modules/kafkajs/src/protocol/requests/deleteGroups/v0/request.js new file mode 100644 index 0000000..c17b7ef --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteGroups/v0/request.js @@ -0,0 +1,22 @@ +const Encoder = require('../../../encoder') +const { DeleteGroups: apiKey } = require('../../apiKeys') + +/** + * DeleteGroups Request (Version: 0) => [groups_names] + * groups_names => STRING + */ + +/** + */ +module.exports = groupIds => ({ + apiKey, + apiVersion: 0, + apiName: 'DeleteGroups', + encode: async () => { + return new Encoder().writeArray(groupIds.map(encodeGroups)) + }, +}) + +const encodeGroups = group => { + return new Encoder().writeString(group) +} diff --git a/node_modules/kafkajs/src/protocol/requests/deleteGroups/v0/response.js b/node_modules/kafkajs/src/protocol/requests/deleteGroups/v0/response.js new file mode 100644 index 0000000..1e45bd9 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteGroups/v0/response.js @@ -0,0 +1,39 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') +/** + * DeleteGroups Response (Version: 0) => throttle_time_ms [results] + * throttle_time_ms => INT32 + * results => group_id error_code + * group_id => STRING + * error_code => INT16 + */ + +const decodeGroup = decoder => ({ + groupId: decoder.readString(), + errorCode: decoder.readInt16(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTimeMs = decoder.readInt32() + const results = decoder.readArray(decodeGroup) + + for (const result of results) { + if (failure(result.errorCode)) { + result.error = createErrorFromCode(result.errorCode) + } + } + return { + throttleTimeMs, + results, + } +} + +const parse = async data => { + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/deleteGroups/v1/request.js b/node_modules/kafkajs/src/protocol/requests/deleteGroups/v1/request.js new file mode 100644 index 0000000..2864e1e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteGroups/v1/request.js @@ -0,0 +1,7 @@ +const requestV0 = require('../v0/request') + +/** + * DeleteGroups Request (Version: 1) + */ + +module.exports = groupIds => Object.assign(requestV0(groupIds), { apiVersion: 1 }) diff --git a/node_modules/kafkajs/src/protocol/requests/deleteGroups/v1/response.js b/node_modules/kafkajs/src/protocol/requests/deleteGroups/v1/response.js new file mode 100644 index 0000000..b70cc2f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteGroups/v1/response.js @@ -0,0 +1,27 @@ +const { parse, decode: decodeV0 } = require('../v0/response') + +/** + * Starting in version 1, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * DeleteGroups Response (Version: 1) => throttle_time_ms [results] + * throttle_time_ms => INT32 + * results => group_id error_code + * group_id => STRING + * error_code => INT16 + */ + +const decode = async rawData => { + const decoded = await decodeV0(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/deleteRecords/index.js b/node_modules/kafkajs/src/protocol/requests/deleteRecords/index.js new file mode 100644 index 0000000..7c6466b --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteRecords/index.js @@ -0,0 +1,17 @@ +const versions = { + 0: ({ topics, timeout }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ topics, timeout }), response: response({ topics }) } + }, + 1: ({ topics, timeout }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ topics, timeout }), response: response({ topics }) } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/deleteRecords/v0/request.js b/node_modules/kafkajs/src/protocol/requests/deleteRecords/v0/request.js new file mode 100644 index 0000000..0544e33 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteRecords/v0/request.js @@ -0,0 +1,30 @@ +const Encoder = require('../../../encoder') +const { DeleteRecords: apiKey } = require('../../apiKeys') + +/** + * DeleteRecords Request (Version: 0) => [topics] timeout_ms + * topics => topic [partitions] + * topic => STRING + * partitions => partition offset + * partition => INT32 + * offset => INT64 + * timeout => INT32 + */ +module.exports = ({ topics, timeout = 5000 }) => ({ + apiKey, + apiVersion: 0, + apiName: 'DeleteRecords', + encode: async () => { + return new Encoder() + .writeArray( + topics.map(({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray( + partitions.map(({ partition, offset }) => { + return new Encoder().writeInt32(partition).writeInt64(offset) + }) + ) + }) + ) + .writeInt32(timeout) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/deleteRecords/v0/response.js b/node_modules/kafkajs/src/protocol/requests/deleteRecords/v0/response.js new file mode 100644 index 0000000..16cc54e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteRecords/v0/response.js @@ -0,0 +1,65 @@ +const Decoder = require('../../../decoder') +const { KafkaJSDeleteTopicRecordsError } = require('../../../../errors') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * DeleteRecords Response (Version: 0) => throttle_time_ms [topics] + * throttle_time_ms => INT32 + * topics => name [partitions] + * name => STRING + * partitions => partition low_watermark error_code + * partition => INT32 + * low_watermark => INT64 + * error_code => INT16 + */ + +const topicNameComparator = (a, b) => a.topic.localeCompare(b.topic) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + throttleTime: decoder.readInt32(), + topics: decoder + .readArray(decoder => ({ + topic: decoder.readString(), + partitions: decoder.readArray(decoder => ({ + partition: decoder.readInt32(), + lowWatermark: decoder.readInt64(), + errorCode: decoder.readInt16(), + })), + })) + .sort(topicNameComparator), + } +} + +const parse = requestTopics => async data => { + const topicsWithErrors = data.topics + .map(({ partitions }) => ({ + partitionsWithErrors: partitions.filter(({ errorCode }) => failure(errorCode)), + })) + .filter(({ partitionsWithErrors }) => partitionsWithErrors.length) + + if (topicsWithErrors.length > 0) { + // at present we only ever request one topic at a time, so can destructure the arrays + const [{ topic }] = data.topics // topic name + const [{ partitions: requestPartitions }] = requestTopics // requested offset(s) + const [{ partitionsWithErrors }] = topicsWithErrors // partition(s) + error(s) + + throw new KafkaJSDeleteTopicRecordsError({ + topic, + partitions: partitionsWithErrors.map(({ partition, errorCode }) => ({ + partition, + error: createErrorFromCode(errorCode), + // attach the original offset from the request, onto the error response + offset: requestPartitions.find(p => p.partition === partition).offset, + })), + }) + } + + return data +} + +module.exports = ({ topics }) => ({ + decode, + parse: parse(topics), +}) diff --git a/node_modules/kafkajs/src/protocol/requests/deleteRecords/v1/request.js b/node_modules/kafkajs/src/protocol/requests/deleteRecords/v1/request.js new file mode 100644 index 0000000..60d6a79 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteRecords/v1/request.js @@ -0,0 +1,13 @@ +const requestV0 = require('../v0/request') + +/** + * DeleteRecords Request (Version: 1) => [topics] timeout_ms + * topics => topic [partitions] + * topic => STRING + * partitions => partition offset + * partition => INT32 + * offset => INT64 + * timeout => INT32 + */ +module.exports = ({ topics, timeout }) => + Object.assign(requestV0({ topics, timeout }), { apiVersion: 1 }) diff --git a/node_modules/kafkajs/src/protocol/requests/deleteRecords/v1/response.js b/node_modules/kafkajs/src/protocol/requests/deleteRecords/v1/response.js new file mode 100644 index 0000000..3eb2d9c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteRecords/v1/response.js @@ -0,0 +1,34 @@ +const responseV0 = require('../v0/response') + +/** + * Starting in version 1, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * DeleteRecords Response (Version: 1) => throttle_time_ms [topics] + * throttle_time_ms => INT32 + * topics => name [partitions] + * name => STRING + * partitions => partition_index low_watermark error_code + * partition_index => INT32 + * low_watermark => INT64 + * error_code => INT16 + */ + +module.exports = ({ topics }) => { + const { parse, decode: decodeV0 } = responseV0({ topics }) + + const decode = async rawData => { + const decoded = await decodeV0(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } + } + + return { + decode, + parse, + } +} diff --git a/node_modules/kafkajs/src/protocol/requests/deleteTopics/index.js b/node_modules/kafkajs/src/protocol/requests/deleteTopics/index.js new file mode 100644 index 0000000..7373407 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteTopics/index.js @@ -0,0 +1,17 @@ +const versions = { + 0: ({ topics, timeout }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ topics, timeout }), response } + }, + 1: ({ topics, timeout }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ topics, timeout }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/deleteTopics/v0/request.js b/node_modules/kafkajs/src/protocol/requests/deleteTopics/v0/request.js new file mode 100644 index 0000000..af7e970 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteTopics/v0/request.js @@ -0,0 +1,16 @@ +const Encoder = require('../../../encoder') +const { DeleteTopics: apiKey } = require('../../apiKeys') + +/** + * DeleteTopics Request (Version: 0) => [topics] timeout + * topics => STRING + * timeout => INT32 + */ +module.exports = ({ topics, timeout = 5000 }) => ({ + apiKey, + apiVersion: 0, + apiName: 'DeleteTopics', + encode: async () => { + return new Encoder().writeArray(topics).writeInt32(timeout) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/deleteTopics/v0/response.js b/node_modules/kafkajs/src/protocol/requests/deleteTopics/v0/response.js new file mode 100644 index 0000000..184efe3 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteTopics/v0/response.js @@ -0,0 +1,37 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * DeleteTopics Response (Version: 0) => [topic_error_codes] + * topic_error_codes => topic error_code + * topic => STRING + * error_code => INT16 + */ + +const topicNameComparator = (a, b) => a.topic.localeCompare(b.topic) + +const topicErrors = decoder => ({ + topic: decoder.readString(), + errorCode: decoder.readInt16(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + topicErrors: decoder.readArray(topicErrors).sort(topicNameComparator), + } +} + +const parse = async data => { + const topicsWithError = data.topicErrors.filter(({ errorCode }) => failure(errorCode)) + if (topicsWithError.length > 0) { + throw createErrorFromCode(topicsWithError[0].errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/deleteTopics/v1/request.js b/node_modules/kafkajs/src/protocol/requests/deleteTopics/v1/request.js new file mode 100644 index 0000000..40fbd4d --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteTopics/v1/request.js @@ -0,0 +1,10 @@ +const requestV0 = require('../v0/request') + +/** + * DeleteTopics Request (Version: 1) => [topics] timeout + * topics => STRING + * timeout => INT32 + */ + +module.exports = ({ topics, timeout }) => + Object.assign(requestV0({ topics, timeout }), { apiVersion: 1 }) diff --git a/node_modules/kafkajs/src/protocol/requests/deleteTopics/v1/response.js b/node_modules/kafkajs/src/protocol/requests/deleteTopics/v1/response.js new file mode 100644 index 0000000..0b8622e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/deleteTopics/v1/response.js @@ -0,0 +1,36 @@ +const Decoder = require('../../../decoder') +const { parse: parseV0 } = require('../v0/response') + +/** + * Starting in version 1, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * DeleteTopics Response (Version: 1) => throttle_time_ms [topic_error_codes] + * throttle_time_ms => INT32 + * topic_error_codes => topic error_code + * topic => STRING + * error_code => INT16 + */ + +const topicNameComparator = (a, b) => a.topic.localeCompare(b.topic) + +const topicErrors = decoder => ({ + topic: decoder.readString(), + errorCode: decoder.readInt16(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + + return { + throttleTime: 0, + clientSideThrottleTime: throttleTime, + topicErrors: decoder.readArray(topicErrors).sort(topicNameComparator), + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/describeAcls/index.js b/node_modules/kafkajs/src/protocol/requests/describeAcls/index.js new file mode 100644 index 0000000..d5aebb1 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeAcls/index.js @@ -0,0 +1,39 @@ +const versions = { + 0: ({ resourceType, resourceName, principal, host, operation, permissionType }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { + request: request({ resourceType, resourceName, principal, host, operation, permissionType }), + response, + } + }, + 1: ({ + resourceType, + resourceName, + resourcePatternType, + principal, + host, + operation, + permissionType, + }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { + request: request({ + resourceType, + resourceName, + resourcePatternType, + principal, + host, + operation, + permissionType, + }), + response, + } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/describeAcls/v0/request.js b/node_modules/kafkajs/src/protocol/requests/describeAcls/v0/request.js new file mode 100644 index 0000000..7786c7e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeAcls/v0/request.js @@ -0,0 +1,27 @@ +const Encoder = require('../../../encoder') +const { DescribeAcls: apiKey } = require('../../apiKeys') + +/** + * DescribeAcls Request (Version: 0) => resource_type resource_name principal host operation permission_type + * resource_type => INT8 + * resource_name => NULLABLE_STRING + * principal => NULLABLE_STRING + * host => NULLABLE_STRING + * operation => INT8 + * permission_type => INT8 + */ + +module.exports = ({ resourceType, resourceName, principal, host, operation, permissionType }) => ({ + apiKey, + apiVersion: 0, + apiName: 'DescribeAcls', + encode: async () => { + return new Encoder() + .writeInt8(resourceType) + .writeString(resourceName) + .writeString(principal) + .writeString(host) + .writeInt8(operation) + .writeInt8(permissionType) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/describeAcls/v0/response.js b/node_modules/kafkajs/src/protocol/requests/describeAcls/v0/response.js new file mode 100644 index 0000000..116268f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeAcls/v0/response.js @@ -0,0 +1,58 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * DescribeAcls Response (Version: 0) => throttle_time_ms error_code error_message [resources] + * throttle_time_ms => INT32 + * error_code => INT16 + * error_message => NULLABLE_STRING + * resources => resource_type resource_name [acls] + * resource_type => INT8 + * resource_name => STRING + * acls => principal host operation permission_type + * principal => STRING + * host => STRING + * operation => INT8 + * permission_type => INT8 + */ + +const decodeAcls = decoder => ({ + principal: decoder.readString(), + host: decoder.readString(), + operation: decoder.readInt8(), + permissionType: decoder.readInt8(), +}) + +const decodeResources = decoder => ({ + resourceType: decoder.readInt8(), + resourceName: decoder.readString(), + acls: decoder.readArray(decodeAcls), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + const errorMessage = decoder.readString() + const resources = decoder.readArray(decodeResources) + + return { + throttleTime, + errorCode, + errorMessage, + resources, + } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/describeAcls/v1/request.js b/node_modules/kafkajs/src/protocol/requests/describeAcls/v1/request.js new file mode 100644 index 0000000..51a78b0 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeAcls/v1/request.js @@ -0,0 +1,37 @@ +const Encoder = require('../../../encoder') +const { DescribeAcls: apiKey } = require('../../apiKeys') + +/** + * DescribeAcls Request (Version: 1) => resource_type resource_name resource_pattern_type_filter principal host operation permission_type + * resource_type => INT8 + * resource_name => NULLABLE_STRING + * resource_pattern_type_filter => INT8 + * principal => NULLABLE_STRING + * host => NULLABLE_STRING + * operation => INT8 + * permission_type => INT8 + */ + +module.exports = ({ + resourceType, + resourceName, + resourcePatternType, + principal, + host, + operation, + permissionType, +}) => ({ + apiKey, + apiVersion: 1, + apiName: 'DescribeAcls', + encode: async () => { + return new Encoder() + .writeInt8(resourceType) + .writeString(resourceName) + .writeInt8(resourcePatternType) + .writeString(principal) + .writeString(host) + .writeInt8(operation) + .writeInt8(permissionType) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/describeAcls/v1/response.js b/node_modules/kafkajs/src/protocol/requests/describeAcls/v1/response.js new file mode 100644 index 0000000..93081cb --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeAcls/v1/response.js @@ -0,0 +1,57 @@ +const { parse } = require('../v0/response') +const Decoder = require('../../../decoder') + +/** + * Starting in version 1, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * Version 1 also introduces a new resource pattern type field. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-290%3A+Support+for+Prefixed+ACLs + * + * DescribeAcls Response (Version: 1) => throttle_time_ms error_code error_message [resources] + * throttle_time_ms => INT32 + * error_code => INT16 + * error_message => NULLABLE_STRING + * resources => resource_type resource_name resource_pattern_type [acls] + * resource_type => INT8 + * resource_name => STRING + * resource_pattern_type => INT8 + * acls => principal host operation permission_type + * principal => STRING + * host => STRING + * operation => INT8 + * permission_type => INT8 + */ +const decodeAcls = decoder => ({ + principal: decoder.readString(), + host: decoder.readString(), + operation: decoder.readInt8(), + permissionType: decoder.readInt8(), +}) + +const decodeResources = decoder => ({ + resourceType: decoder.readInt8(), + resourceName: decoder.readString(), + resourcePatternType: decoder.readInt8(), + acls: decoder.readArray(decodeAcls), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + const errorMessage = decoder.readString() + const resources = decoder.readArray(decodeResources) + + return { + throttleTime: 0, + clientSideThrottleTime: throttleTime, + errorCode, + errorMessage, + resources, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/describeConfigs/index.js b/node_modules/kafkajs/src/protocol/requests/describeConfigs/index.js new file mode 100644 index 0000000..809d729 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeConfigs/index.js @@ -0,0 +1,22 @@ +const versions = { + 0: ({ resources }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ resources }), response } + }, + 1: ({ resources, includeSynonyms }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ resources, includeSynonyms }), response } + }, + 2: ({ resources, includeSynonyms }) => { + const request = require('./v2/request') + const response = require('./v2/response') + return { request: request({ resources, includeSynonyms }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/describeConfigs/v0/request.js b/node_modules/kafkajs/src/protocol/requests/describeConfigs/v0/request.js new file mode 100644 index 0000000..2463573 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeConfigs/v0/request.js @@ -0,0 +1,29 @@ +const Encoder = require('../../../encoder') +const { DescribeConfigs: apiKey } = require('../../apiKeys') + +/** + * DescribeConfigs Request (Version: 0) => [resources] + * resources => resource_type resource_name [config_names] + * resource_type => INT8 + * resource_name => STRING + * config_names => STRING + */ + +/** + * @param {Array} resources An array of config resources to be returned + */ +module.exports = ({ resources }) => ({ + apiKey, + apiVersion: 0, + apiName: 'DescribeConfigs', + encode: async () => { + return new Encoder().writeArray(resources.map(encodeResource)) + }, +}) + +const encodeResource = ({ type, name, configNames = [] }) => { + return new Encoder() + .writeInt8(type) + .writeString(name) + .writeNullableArray(configNames) +} diff --git a/node_modules/kafkajs/src/protocol/requests/describeConfigs/v0/response.js b/node_modules/kafkajs/src/protocol/requests/describeConfigs/v0/response.js new file mode 100644 index 0000000..149509c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeConfigs/v0/response.js @@ -0,0 +1,98 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') +const ConfigSource = require('../../../configSource') +const ConfigResourceTypes = require('../../../configResourceTypes') + +/** + * DescribeConfigs Response (Version: 0) => throttle_time_ms [resources] + * throttle_time_ms => INT32 + * resources => error_code error_message resource_type resource_name [config_entries] + * error_code => INT16 + * error_message => NULLABLE_STRING + * resource_type => INT8 + * resource_name => STRING + * config_entries => config_name config_value read_only is_default is_sensitive + * config_name => STRING + * config_value => NULLABLE_STRING + * read_only => BOOLEAN + * is_default => BOOLEAN + * is_sensitive => BOOLEAN + */ + +const decodeConfigEntries = (decoder, resourceType) => { + const configName = decoder.readString() + const configValue = decoder.readString() + const readOnly = decoder.readBoolean() + const isDefault = decoder.readBoolean() + const isSensitive = decoder.readBoolean() + + /** + * Backporting ConfigSource value to v0 + * @see https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/requests/DescribeConfigsResponse.java#L232-L242 + */ + let configSource + if (isDefault) { + configSource = ConfigSource.DEFAULT_CONFIG + } else { + switch (resourceType) { + case ConfigResourceTypes.BROKER: + configSource = ConfigSource.STATIC_BROKER_CONFIG + break + case ConfigResourceTypes.TOPIC: + configSource = ConfigSource.TOPIC_CONFIG + break + default: + configSource = ConfigSource.UNKNOWN + } + } + + return { + configName, + configValue, + readOnly, + isDefault, + configSource, + isSensitive, + } +} + +const decodeResources = decoder => { + const errorCode = decoder.readInt16() + const errorMessage = decoder.readString() + const resourceType = decoder.readInt8() + const resourceName = decoder.readString() + const configEntries = decoder.readArray(decoder => decodeConfigEntries(decoder, resourceType)) + + return { + errorCode, + errorMessage, + resourceType, + resourceName, + configEntries, + } +} + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const resources = decoder.readArray(decodeResources) + + return { + throttleTime, + resources, + } +} + +const parse = async data => { + const resourcesWithError = data.resources.filter(({ errorCode }) => failure(errorCode)) + if (resourcesWithError.length > 0) { + throw createErrorFromCode(resourcesWithError[0].errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/describeConfigs/v1/request.js b/node_modules/kafkajs/src/protocol/requests/describeConfigs/v1/request.js new file mode 100644 index 0000000..4f38674 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeConfigs/v1/request.js @@ -0,0 +1,31 @@ +const Encoder = require('../../../encoder') +const { DescribeConfigs: apiKey } = require('../../apiKeys') + +/** + * DescribeConfigs Request (Version: 1) => [resources] include_synonyms + * resources => resource_type resource_name [config_names] + * resource_type => INT8 + * resource_name => STRING + * config_names => STRING + * include_synonyms => BOOLEAN + */ + +/** + * @param {Array} resources An array of config resources to be returned + * @param [includeSynonyms=false] + */ +module.exports = ({ resources, includeSynonyms = false }) => ({ + apiKey, + apiVersion: 1, + apiName: 'DescribeConfigs', + encode: async () => { + return new Encoder().writeArray(resources.map(encodeResource)).writeBoolean(includeSynonyms) + }, +}) + +const encodeResource = ({ type, name, configNames = [] }) => { + return new Encoder() + .writeInt8(type) + .writeString(name) + .writeNullableArray(configNames) +} diff --git a/node_modules/kafkajs/src/protocol/requests/describeConfigs/v1/response.js b/node_modules/kafkajs/src/protocol/requests/describeConfigs/v1/response.js new file mode 100644 index 0000000..7c7e8cc --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeConfigs/v1/response.js @@ -0,0 +1,72 @@ +const Decoder = require('../../../decoder') +const { parse: parseV0 } = require('../v0/response') +const { DEFAULT_CONFIG } = require('../../../configSource') + +/** + * DescribeConfigs Response (Version: 1) => throttle_time_ms [resources] + * throttle_time_ms => INT32 + * resources => error_code error_message resource_type resource_name [config_entries] + * error_code => INT16 + * error_message => NULLABLE_STRING + * resource_type => INT8 + * resource_name => STRING + * config_entries => config_name config_value read_only config_source is_sensitive [config_synonyms] + * config_name => STRING + * config_value => NULLABLE_STRING + * read_only => BOOLEAN + * config_source => INT8 + * is_sensitive => BOOLEAN + * config_synonyms => config_name config_value config_source + * config_name => STRING + * config_value => NULLABLE_STRING + * config_source => INT8 + */ + +const decodeSynonyms = decoder => ({ + configName: decoder.readString(), + configValue: decoder.readString(), + configSource: decoder.readInt8(), +}) + +const decodeConfigEntries = decoder => { + const configName = decoder.readString() + const configValue = decoder.readString() + const readOnly = decoder.readBoolean() + const configSource = decoder.readInt8() + const isSensitive = decoder.readBoolean() + const configSynonyms = decoder.readArray(decodeSynonyms) + + return { + configName, + configValue, + readOnly, + isDefault: configSource === DEFAULT_CONFIG, + configSource, + isSensitive, + configSynonyms, + } +} + +const decodeResources = decoder => ({ + errorCode: decoder.readInt16(), + errorMessage: decoder.readString(), + resourceType: decoder.readInt8(), + resourceName: decoder.readString(), + configEntries: decoder.readArray(decodeConfigEntries), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const resources = decoder.readArray(decodeResources) + + return { + throttleTime, + resources, + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/describeConfigs/v2/request.js b/node_modules/kafkajs/src/protocol/requests/describeConfigs/v2/request.js new file mode 100644 index 0000000..65a5bd5 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeConfigs/v2/request.js @@ -0,0 +1,17 @@ +const requestV1 = require('../v1/request') + +/** + * DescribeConfigs Request (Version: 1) => [resources] include_synonyms + * resources => resource_type resource_name [config_names] + * resource_type => INT8 + * resource_name => STRING + * config_names => STRING + * include_synonyms => BOOLEAN + */ + +/** + * @param {Array} resources An array of config resources to be returned + * @param [includeSynonyms=false] + */ +module.exports = ({ resources, includeSynonyms }) => + Object.assign(requestV1({ resources, includeSynonyms }), { apiVersion: 2 }) diff --git a/node_modules/kafkajs/src/protocol/requests/describeConfigs/v2/response.js b/node_modules/kafkajs/src/protocol/requests/describeConfigs/v2/response.js new file mode 100644 index 0000000..b54e880 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeConfigs/v2/response.js @@ -0,0 +1,39 @@ +const { parse, decode: decodeV1 } = require('../v1/response') + +/** + * Starting in version 2, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * DescribeConfigs Response (Version: 2) => throttle_time_ms [resources] + * throttle_time_ms => INT32 + * resources => error_code error_message resource_type resource_name [config_entries] + * error_code => INT16 + * error_message => NULLABLE_STRING + * resource_type => INT8 + * resource_name => STRING + * config_entries => config_name config_value read_only config_source is_sensitive [config_synonyms] + * config_name => STRING + * config_value => NULLABLE_STRING + * read_only => BOOLEAN + * config_source => INT8 + * is_sensitive => BOOLEAN + * config_synonyms => config_name config_value config_source + * config_name => STRING + * config_value => NULLABLE_STRING + * config_source => INT8 + */ + +const decode = async rawData => { + const decoded = await decodeV1(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/describeGroups/index.js b/node_modules/kafkajs/src/protocol/requests/describeGroups/index.js new file mode 100644 index 0000000..a139d71 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeGroups/index.js @@ -0,0 +1,22 @@ +const versions = { + 0: ({ groupIds }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ groupIds }), response } + }, + 1: ({ groupIds }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ groupIds }), response } + }, + 2: ({ groupIds }) => { + const request = require('./v2/request') + const response = require('./v2/response') + return { request: request({ groupIds }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/describeGroups/v0/request.js b/node_modules/kafkajs/src/protocol/requests/describeGroups/v0/request.js new file mode 100644 index 0000000..e2d8331 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeGroups/v0/request.js @@ -0,0 +1,19 @@ +const Encoder = require('../../../encoder') +const { DescribeGroups: apiKey } = require('../../apiKeys') + +/** + * DescribeGroups Request (Version: 0) => [group_ids] + * group_ids => STRING + */ + +/** + * @param {Array} groupIds List of groupIds to request metadata for (an empty groupId array will return empty group metadata) + */ +module.exports = ({ groupIds }) => ({ + apiKey, + apiVersion: 0, + apiName: 'DescribeGroups', + encode: async () => { + return new Encoder().writeArray(groupIds) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/describeGroups/v0/response.js b/node_modules/kafkajs/src/protocol/requests/describeGroups/v0/response.js new file mode 100644 index 0000000..aec374e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeGroups/v0/response.js @@ -0,0 +1,58 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * DescribeGroups Response (Version: 0) => [groups] + * groups => error_code group_id state protocol_type protocol [members] + * error_code => INT16 + * group_id => STRING + * state => STRING + * protocol_type => STRING + * protocol => STRING + * members => member_id client_id client_host member_metadata member_assignment + * member_id => STRING + * client_id => STRING + * client_host => STRING + * member_metadata => BYTES + * member_assignment => BYTES + */ + +const decoderMember = decoder => ({ + memberId: decoder.readString(), + clientId: decoder.readString(), + clientHost: decoder.readString(), + memberMetadata: decoder.readBytes(), + memberAssignment: decoder.readBytes(), +}) + +const decodeGroup = decoder => ({ + errorCode: decoder.readInt16(), + groupId: decoder.readString(), + state: decoder.readString(), + protocolType: decoder.readString(), + protocol: decoder.readString(), + members: decoder.readArray(decoderMember), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const groups = decoder.readArray(decodeGroup) + + return { + groups, + } +} + +const parse = async data => { + const groupsWithError = data.groups.filter(({ errorCode }) => failure(errorCode)) + if (groupsWithError.length > 0) { + throw createErrorFromCode(groupsWithError[0].errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/describeGroups/v1/request.js b/node_modules/kafkajs/src/protocol/requests/describeGroups/v1/request.js new file mode 100644 index 0000000..93eddcb --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeGroups/v1/request.js @@ -0,0 +1,8 @@ +const requestV0 = require('../v0/request') + +/** + * DescribeGroups Request (Version: 1) => [group_ids] + * group_ids => STRING + */ + +module.exports = ({ groupIds }) => Object.assign(requestV0({ groupIds }), { apiVersion: 1 }) diff --git a/node_modules/kafkajs/src/protocol/requests/describeGroups/v1/response.js b/node_modules/kafkajs/src/protocol/requests/describeGroups/v1/response.js new file mode 100644 index 0000000..5f0cadd --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeGroups/v1/response.js @@ -0,0 +1,52 @@ +const Decoder = require('../../../decoder') +const { parse: parseV0 } = require('../v0/response') + +/** + * DescribeGroups Response (Version: 1) => throttle_time_ms [groups] + * throttle_time_ms => INT32 + * groups => error_code group_id state protocol_type protocol [members] + * error_code => INT16 + * group_id => STRING + * state => STRING + * protocol_type => STRING + * protocol => STRING + * members => member_id client_id client_host member_metadata member_assignment + * member_id => STRING + * client_id => STRING + * client_host => STRING + * member_metadata => BYTES + * member_assignment => BYTES + */ + +const decoderMember = decoder => ({ + memberId: decoder.readString(), + clientId: decoder.readString(), + clientHost: decoder.readString(), + memberMetadata: decoder.readBytes(), + memberAssignment: decoder.readBytes(), +}) + +const decodeGroup = decoder => ({ + errorCode: decoder.readInt16(), + groupId: decoder.readString(), + state: decoder.readString(), + protocolType: decoder.readString(), + protocol: decoder.readString(), + members: decoder.readArray(decoderMember), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const groups = decoder.readArray(decodeGroup) + + return { + throttleTime, + groups, + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/describeGroups/v2/request.js b/node_modules/kafkajs/src/protocol/requests/describeGroups/v2/request.js new file mode 100644 index 0000000..5ae1dd2 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeGroups/v2/request.js @@ -0,0 +1,8 @@ +const requestV1 = require('../v1/request') + +/** + * DescribeGroups Request (Version: 2) => [group_ids] + * group_ids => STRING + */ + +module.exports = ({ groupIds }) => Object.assign(requestV1({ groupIds }), { apiVersion: 2 }) diff --git a/node_modules/kafkajs/src/protocol/requests/describeGroups/v2/response.js b/node_modules/kafkajs/src/protocol/requests/describeGroups/v2/response.js new file mode 100644 index 0000000..6ed7964 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/describeGroups/v2/response.js @@ -0,0 +1,36 @@ +const { parse, decode: decodeV1 } = require('../v1/response') + +/** + * Starting in version 2, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * DescribeGroups Response (Version: 2) => throttle_time_ms [groups] + * throttle_time_ms => INT32 + * groups => error_code group_id state protocol_type protocol [members] + * error_code => INT16 + * group_id => STRING + * state => STRING + * protocol_type => STRING + * protocol => STRING + * members => member_id client_id client_host member_metadata member_assignment + * member_id => STRING + * client_id => STRING + * client_host => STRING + * member_metadata => BYTES + * member_assignment => BYTES + */ + +const decode = async rawData => { + const decoded = await decodeV1(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/endTxn/index.js b/node_modules/kafkajs/src/protocol/requests/endTxn/index.js new file mode 100644 index 0000000..66a93ed --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/endTxn/index.js @@ -0,0 +1,23 @@ +const versions = { + 0: ({ transactionalId, producerId, producerEpoch, transactionResult }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { + request: request({ transactionalId, producerId, producerEpoch, transactionResult }), + response, + } + }, + 1: ({ transactionalId, producerId, producerEpoch, transactionResult }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { + request: request({ transactionalId, producerId, producerEpoch, transactionResult }), + response, + } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/endTxn/v0/request.js b/node_modules/kafkajs/src/protocol/requests/endTxn/v0/request.js new file mode 100644 index 0000000..fb3d118 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/endTxn/v0/request.js @@ -0,0 +1,23 @@ +const Encoder = require('../../../encoder') +const { EndTxn: apiKey } = require('../../apiKeys') + +/** + * EndTxn Request (Version: 0) => transactional_id producer_id producer_epoch transaction_result + * transactional_id => STRING + * producer_id => INT64 + * producer_epoch => INT16 + * transaction_result => BOOLEAN + */ + +module.exports = ({ transactionalId, producerId, producerEpoch, transactionResult }) => ({ + apiKey, + apiVersion: 0, + apiName: 'EndTxn', + encode: async () => { + return new Encoder() + .writeString(transactionalId) + .writeInt64(producerId) + .writeInt16(producerEpoch) + .writeBoolean(transactionResult) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/endTxn/v0/response.js b/node_modules/kafkajs/src/protocol/requests/endTxn/v0/response.js new file mode 100644 index 0000000..f6db2fe --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/endTxn/v0/response.js @@ -0,0 +1,33 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode, failIfVersionNotSupported } = require('../../../error') + +/** + * EndTxn Response (Version: 0) => throttle_time_ms error_code + * throttle_time_ms => INT32 + * error_code => INT16 + */ +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { + throttleTime, + errorCode, + } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/endTxn/v1/request.js b/node_modules/kafkajs/src/protocol/requests/endTxn/v1/request.js new file mode 100644 index 0000000..39d8d37 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/endTxn/v1/request.js @@ -0,0 +1,14 @@ +const requestV0 = require('../v0/request') + +/** + * EndTxn Request (Version: 1) => transactional_id producer_id producer_epoch transaction_result + * transactional_id => STRING + * producer_id => INT64 + * producer_epoch => INT16 + * transaction_result => BOOLEAN + */ + +module.exports = ({ transactionalId, producerId, producerEpoch, transactionResult }) => + Object.assign(requestV0({ transactionalId, producerId, producerEpoch, transactionResult }), { + apiVersion: 1, + }) diff --git a/node_modules/kafkajs/src/protocol/requests/endTxn/v1/response.js b/node_modules/kafkajs/src/protocol/requests/endTxn/v1/response.js new file mode 100644 index 0000000..a2ba807 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/endTxn/v1/response.js @@ -0,0 +1,25 @@ +const { parse, decode: decodeV0 } = require('../v0/response') + +/** + * Starting in version 1, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * EndTxn Response (Version: 1) => throttle_time_ms error_code + * throttle_time_ms => INT32 + * error_code => INT16 + */ + +const decode = async rawData => { + const decoded = await decodeV0(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/index.js b/node_modules/kafkajs/src/protocol/requests/fetch/index.js new file mode 100644 index 0000000..55d2ee6 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/index.js @@ -0,0 +1,251 @@ +const ISOLATION_LEVEL = require('../../isolationLevel') + +// For normal consumers, use -1 +const REPLICA_ID = -1 +const NETWORK_DELAY = 100 + +/** + * The FETCH request can block up to maxWaitTime, which can be bigger than the configured + * request timeout. It's safer to always use the maxWaitTime + **/ +const requestTimeout = timeout => + Number.isSafeInteger(timeout + NETWORK_DELAY) ? timeout + NETWORK_DELAY : timeout + +const versions = { + 0: ({ replicaId = REPLICA_ID, maxWaitTime, minBytes, topics }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { + request: request({ replicaId, maxWaitTime, minBytes, topics }), + response, + requestTimeout: requestTimeout(maxWaitTime), + } + }, + 1: ({ replicaId = REPLICA_ID, maxWaitTime, minBytes, topics }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { + request: request({ replicaId, maxWaitTime, minBytes, topics }), + response, + requestTimeout: requestTimeout(maxWaitTime), + } + }, + 2: ({ replicaId = REPLICA_ID, maxWaitTime, minBytes, topics }) => { + const request = require('./v2/request') + const response = require('./v2/response') + return { + request: request({ replicaId, maxWaitTime, minBytes, topics }), + response, + requestTimeout: requestTimeout(maxWaitTime), + } + }, + 3: ({ replicaId = REPLICA_ID, maxWaitTime, minBytes, maxBytes, topics }) => { + const request = require('./v3/request') + const response = require('./v3/response') + return { + request: request({ replicaId, maxWaitTime, minBytes, maxBytes, topics }), + response, + requestTimeout: requestTimeout(maxWaitTime), + } + }, + 4: ({ + replicaId = REPLICA_ID, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + maxWaitTime, + minBytes, + maxBytes, + topics, + }) => { + const request = require('./v4/request') + const response = require('./v4/response') + return { + request: request({ replicaId, isolationLevel, maxWaitTime, minBytes, maxBytes, topics }), + response, + requestTimeout: requestTimeout(maxWaitTime), + } + }, + 5: ({ + replicaId = REPLICA_ID, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + maxWaitTime, + minBytes, + maxBytes, + topics, + }) => { + const request = require('./v5/request') + const response = require('./v5/response') + return { + request: request({ replicaId, isolationLevel, maxWaitTime, minBytes, maxBytes, topics }), + response, + requestTimeout: requestTimeout(maxWaitTime), + } + }, + 6: ({ + replicaId = REPLICA_ID, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + maxWaitTime, + minBytes, + maxBytes, + topics, + }) => { + const request = require('./v6/request') + const response = require('./v6/response') + return { + request: request({ replicaId, isolationLevel, maxWaitTime, minBytes, maxBytes, topics }), + response, + requestTimeout: requestTimeout(maxWaitTime), + } + }, + 7: ({ + replicaId = REPLICA_ID, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + sessionId = 0, + sessionEpoch = -1, + forgottenTopics = [], + maxWaitTime, + minBytes, + maxBytes, + topics, + }) => { + const request = require('./v7/request') + const response = require('./v7/response') + return { + request: request({ + replicaId, + isolationLevel, + sessionId, + sessionEpoch, + forgottenTopics, + maxWaitTime, + minBytes, + maxBytes, + topics, + }), + response, + requestTimeout: requestTimeout(maxWaitTime), + } + }, + 8: ({ + replicaId = REPLICA_ID, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + sessionId = 0, + sessionEpoch = -1, + forgottenTopics = [], + maxWaitTime, + minBytes, + maxBytes, + topics, + }) => { + const request = require('./v8/request') + const response = require('./v8/response') + return { + request: request({ + replicaId, + isolationLevel, + sessionId, + sessionEpoch, + forgottenTopics, + maxWaitTime, + minBytes, + maxBytes, + topics, + }), + response, + requestTimeout: requestTimeout(maxWaitTime), + } + }, + 9: ({ + replicaId = REPLICA_ID, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + sessionId = 0, + sessionEpoch = -1, + forgottenTopics = [], + maxWaitTime, + minBytes, + maxBytes, + topics, + }) => { + const request = require('./v9/request') + const response = require('./v9/response') + return { + request: request({ + replicaId, + isolationLevel, + sessionId, + sessionEpoch, + forgottenTopics, + maxWaitTime, + minBytes, + maxBytes, + topics, + }), + response, + requestTimeout: requestTimeout(maxWaitTime), + } + }, + 10: ({ + replicaId = REPLICA_ID, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + sessionId = 0, + sessionEpoch = -1, + forgottenTopics = [], + maxWaitTime, + minBytes, + maxBytes, + topics, + }) => { + const request = require('./v10/request') + const response = require('./v10/response') + return { + request: request({ + replicaId, + isolationLevel, + sessionId, + sessionEpoch, + forgottenTopics, + maxWaitTime, + minBytes, + maxBytes, + topics, + }), + response, + requestTimeout: requestTimeout(maxWaitTime), + } + }, + 11: ({ + replicaId = REPLICA_ID, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + sessionId = 0, + sessionEpoch = -1, + forgottenTopics = [], + maxWaitTime, + minBytes, + maxBytes, + topics, + rackId, + }) => { + const request = require('./v11/request') + const response = require('./v11/response') + return { + request: request({ + replicaId, + isolationLevel, + sessionId, + sessionEpoch, + forgottenTopics, + maxWaitTime, + minBytes, + maxBytes, + topics, + rackId, + }), + response, + requestTimeout: requestTimeout(maxWaitTime), + } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v0/request.js b/node_modules/kafkajs/src/protocol/requests/fetch/v0/request.js new file mode 100644 index 0000000..79bf728 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v0/request.js @@ -0,0 +1,57 @@ +const Encoder = require('../../../encoder') +const { Fetch: apiKey } = require('../../apiKeys') + +/** + * Fetch Request (Version: 0) => replica_id max_wait_time min_bytes [topics] + * replica_id => INT32 + * max_wait_time => INT32 + * min_bytes => INT32 + * topics => topic [partitions] + * topic => STRING + * partitions => partition fetch_offset max_bytes + * partition => INT32 + * fetch_offset => INT64 + * max_bytes => INT32 + */ + +/** + * @param {number} replicaId Broker id of the follower + * @param {number} maxWaitTime Maximum time in ms to wait for the response + * @param {number} minBytes Minimum bytes to accumulate in the response. + * @param {Array} topics Topics to fetch + * [ + * { + * topic: 'topic-name', + * partitions: [ + * { + * partition: 0, + * fetchOffset: '4124', + * maxBytes: 2048 + * } + * ] + * } + * ] + */ +module.exports = ({ replicaId, maxWaitTime, minBytes, topics }) => ({ + apiKey, + apiVersion: 0, + apiName: 'Fetch', + encode: async () => { + return new Encoder() + .writeInt32(replicaId) + .writeInt32(maxWaitTime) + .writeInt32(minBytes) + .writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition, fetchOffset, maxBytes }) => { + return new Encoder() + .writeInt32(partition) + .writeInt64(fetchOffset) + .writeInt32(maxBytes) +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v0/response.js b/node_modules/kafkajs/src/protocol/requests/fetch/v0/response.js new file mode 100644 index 0000000..4a83a6d --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v0/response.js @@ -0,0 +1,65 @@ +const Decoder = require('../../../decoder') +const { KafkaJSOffsetOutOfRange } = require('../../../../errors') +const { failure, createErrorFromCode, errorCodes } = require('../../../error') +const MessageSetDecoder = require('../../../messageSet/decoder') + +/** + * Fetch Response (Version: 0) => [responses] + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition_header record_set + * partition_header => partition error_code high_watermark + * partition => INT32 + * error_code => INT16 + * high_watermark => INT64 + * record_set => RECORDS + */ + +const decodePartition = async decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + highWatermark: decoder.readInt64().toString(), + messages: await MessageSetDecoder(decoder), +}) + +const decodeResponse = async decoder => ({ + topicName: decoder.readString(), + partitions: await decoder.readArrayAsync(decodePartition), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const responses = await decoder.readArrayAsync(decodeResponse) + + return { + responses, + } +} + +const { code: OFFSET_OUT_OF_RANGE_ERROR_CODE } = errorCodes.find( + e => e.type === 'OFFSET_OUT_OF_RANGE' +) + +const parse = async data => { + const errors = data.responses.flatMap(({ topicName, partitions }) => { + return partitions + .filter(partition => failure(partition.errorCode)) + .map(partition => Object.assign({}, partition, { topic: topicName })) + }) + + if (errors.length > 0) { + const { errorCode, topic, partition } = errors[0] + if (errorCode === OFFSET_OUT_OF_RANGE_ERROR_CODE) { + throw new KafkaJSOffsetOutOfRange(createErrorFromCode(errorCode), { topic, partition }) + } + + throw createErrorFromCode(errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v1/request.js b/node_modules/kafkajs/src/protocol/requests/fetch/v1/request.js new file mode 100644 index 0000000..a555b15 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v1/request.js @@ -0,0 +1,5 @@ +const requestV0 = require('../v0/request') + +module.exports = ({ replicaId, maxWaitTime, minBytes, topics }) => { + return Object.assign(requestV0({ replicaId, maxWaitTime, minBytes, topics }), { apiVersion: 1 }) +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v1/response.js b/node_modules/kafkajs/src/protocol/requests/fetch/v1/response.js new file mode 100644 index 0000000..62c825e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v1/response.js @@ -0,0 +1,44 @@ +const Decoder = require('../../../decoder') +const { parse: parseV0 } = require('../v0/response') +const MessageSetDecoder = require('../../../messageSet/decoder') + +/** + * Fetch Response (Version: 1) => throttle_time_ms [responses] + * throttle_time_ms => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition_header record_set + * partition_header => partition error_code high_watermark + * partition => INT32 + * error_code => INT16 + * high_watermark => INT64 + * record_set => RECORDS + */ + +const decodePartition = async decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + highWatermark: decoder.readInt64().toString(), + messages: await MessageSetDecoder(decoder), +}) + +const decodeResponse = async decoder => ({ + topicName: decoder.readString(), + partitions: await decoder.readArrayAsync(decodePartition), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const responses = await decoder.readArrayAsync(decodeResponse) + + return { + throttleTime, + responses, + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v10/request.js b/node_modules/kafkajs/src/protocol/requests/fetch/v10/request.js new file mode 100644 index 0000000..09c1f37 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v10/request.js @@ -0,0 +1,55 @@ +const ISOLATION_LEVEL = require('../../../isolationLevel') +const requestV9 = require('../v9/request') + +/** + * ZStd Compression + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-110%3A+Add+Codec+for+ZStandard+Compression + */ + +/** + * Fetch Request (Version: 10) => replica_id max_wait_time min_bytes max_bytes isolation_level session_id session_epoch [topics] [forgotten_topics_data] + * replica_id => INT32 + * max_wait_time => INT32 + * min_bytes => INT32 + * max_bytes => INT32 + * isolation_level => INT8 + * session_id => INT32 + * session_epoch => INT32 + * topics => topic [partitions] + * topic => STRING + * partitions => partition current_leader_epoch fetch_offset log_start_offset partition_max_bytes + * partition => INT32 + * current_leader_epoch => INT32 + * fetch_offset => INT64 + * log_start_offset => INT64 + * partition_max_bytes => INT32 + * forgotten_topics_data => topic [partitions] + * topic => STRING + * partitions => INT32 + */ + +module.exports = ({ + replicaId, + maxWaitTime, + minBytes, + maxBytes, + topics, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + sessionId = 0, + sessionEpoch = -1, + forgottenTopics = [], // Topics to remove from the fetch session +}) => + Object.assign( + requestV9({ + replicaId, + maxWaitTime, + minBytes, + maxBytes, + topics, + isolationLevel, + sessionId, + sessionEpoch, + forgottenTopics, + }), + { apiVersion: 10 } + ) diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v10/response.js b/node_modules/kafkajs/src/protocol/requests/fetch/v10/response.js new file mode 100644 index 0000000..c09a298 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v10/response.js @@ -0,0 +1,26 @@ +const { decode, parse } = require('../v9/response') + +/** + * Fetch Response (Version: 10) => throttle_time_ms error_code session_id [responses] + * throttle_time_ms => INT32 + * error_code => INT16 + * session_id => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition_header record_set + * partition_header => partition error_code high_watermark last_stable_offset log_start_offset [aborted_transactions] + * partition => INT32 + * error_code => INT16 + * high_watermark => INT64 + * last_stable_offset => INT64 + * log_start_offset => INT64 + * aborted_transactions => producer_id first_offset + * producer_id => INT64 + * first_offset => INT64 + * record_set => RECORDS + */ + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v11/request.js b/node_modules/kafkajs/src/protocol/requests/fetch/v11/request.js new file mode 100644 index 0000000..efd797b --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v11/request.js @@ -0,0 +1,84 @@ +const Encoder = require('../../../encoder') +const { Fetch: apiKey } = require('../../apiKeys') +const ISOLATION_LEVEL = require('../../../isolationLevel') + +/** + * Allow consumers to fetch from closest replica + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-392%3A+Allow+consumers+to+fetch+from+closest+replica + */ + +/** + * Fetch Request (Version: 11) => replica_id max_wait_time min_bytes max_bytes isolation_level session_id session_epoch [topics] [forgotten_topics_data] + * replica_id => INT32 + * max_wait_time => INT32 + * min_bytes => INT32 + * max_bytes => INT32 + * isolation_level => INT8 + * session_id => INT32 + * session_epoch => INT32 + * topics => topic [partitions] + * topic => STRING + * partitions => partition current_leader_epoch fetch_offset log_start_offset partition_max_bytes + * partition => INT32 + * current_leader_epoch => INT32 + * fetch_offset => INT64 + * log_start_offset => INT64 + * partition_max_bytes => INT32 + * forgotten_topics_data => topic [partitions] + * topic => STRING + * partitions => INT32 + * rack_id => STRING + */ + +module.exports = ({ + replicaId, + maxWaitTime, + minBytes, + maxBytes, + topics, + rackId = '', + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + sessionId = 0, + sessionEpoch = -1, + forgottenTopics = [], // Topics to remove from the fetch session +}) => ({ + apiKey, + apiVersion: 11, + apiName: 'Fetch', + encode: async () => { + return new Encoder() + .writeInt32(replicaId) + .writeInt32(maxWaitTime) + .writeInt32(minBytes) + .writeInt32(maxBytes) + .writeInt8(isolationLevel) + .writeInt32(sessionId) + .writeInt32(sessionEpoch) + .writeArray(topics.map(encodeTopic)) + .writeArray(forgottenTopics.map(encodeForgottenTopics)) + .writeString(rackId) + }, +}) + +const encodeForgottenTopics = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions) +} + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ + partition, + currentLeaderEpoch = -1, + fetchOffset, + logStartOffset = -1, + maxBytes, +}) => { + return new Encoder() + .writeInt32(partition) + .writeInt32(currentLeaderEpoch) + .writeInt64(fetchOffset) + .writeInt64(logStartOffset) + .writeInt32(maxBytes) +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v11/response.js b/node_modules/kafkajs/src/protocol/requests/fetch/v11/response.js new file mode 100644 index 0000000..e23f913 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v11/response.js @@ -0,0 +1,69 @@ +const Decoder = require('../../../decoder') +const { parse: parseV1 } = require('../v1/response') +const decodeMessages = require('../v4/decodeMessages') + +/** + * Fetch Response (Version: 11) => throttle_time_ms error_code session_id [responses] + * throttle_time_ms => INT32 + * error_code => INT16 + * session_id => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition_header record_set + * partition_header => partition error_code high_watermark last_stable_offset log_start_offset [aborted_transactions] + * partition => INT32 + * error_code => INT16 + * high_watermark => INT64 + * last_stable_offset => INT64 + * log_start_offset => INT64 + * aborted_transactions => producer_id first_offset + * producer_id => INT64 + * first_offset => INT64 + * preferred_read_replica => INT32 + * record_set => RECORDS + */ + +const decodeAbortedTransactions = decoder => ({ + producerId: decoder.readInt64().toString(), + firstOffset: decoder.readInt64().toString(), +}) + +const decodePartition = async decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + highWatermark: decoder.readInt64().toString(), + lastStableOffset: decoder.readInt64().toString(), + lastStartOffset: decoder.readInt64().toString(), + abortedTransactions: decoder.readArray(decodeAbortedTransactions), + preferredReadReplica: decoder.readInt32(), + messages: await decodeMessages(decoder), +}) + +const decodeResponse = async decoder => ({ + topicName: decoder.readString(), + partitions: await decoder.readArrayAsync(decodePartition), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const clientSideThrottleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + const sessionId = decoder.readInt32() + const responses = await decoder.readArrayAsync(decodeResponse) + + // Report a `throttleTime` of 0: The broker will not have throttled + // this request, but if the `clientSideThrottleTime` is >0 then it + // expects us to do that -- and it will ignore requests. + return { + throttleTime: 0, + clientSideThrottleTime, + errorCode, + sessionId, + responses, + } +} + +module.exports = { + decode, + parse: parseV1, +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v2/request.js b/node_modules/kafkajs/src/protocol/requests/fetch/v2/request.js new file mode 100644 index 0000000..9d4818a --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v2/request.js @@ -0,0 +1,5 @@ +const requestV0 = require('../v0/request') + +module.exports = ({ replicaId, maxWaitTime, minBytes, topics }) => { + return Object.assign(requestV0({ replicaId, maxWaitTime, minBytes, topics }), { apiVersion: 2 }) +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v2/response.js b/node_modules/kafkajs/src/protocol/requests/fetch/v2/response.js new file mode 100644 index 0000000..547fd84 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v2/response.js @@ -0,0 +1,19 @@ +const { decode, parse } = require('../v1/response') + +/** + * Fetch Response (Version: 2) => throttle_time_ms [responses] + * throttle_time_ms => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition_header record_set + * partition_header => partition error_code high_watermark + * partition => INT32 + * error_code => INT16 + * high_watermark => INT64 + * record_set => RECORDS + */ + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v3/request.js b/node_modules/kafkajs/src/protocol/requests/fetch/v3/request.js new file mode 100644 index 0000000..b5ffde8 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v3/request.js @@ -0,0 +1,62 @@ +const Encoder = require('../../../encoder') +const { Fetch: apiKey } = require('../../apiKeys') + +/** + * Fetch Request (Version: 3) => replica_id max_wait_time min_bytes max_bytes [topics] + * replica_id => INT32 + * max_wait_time => INT32 + * min_bytes => INT32 + * max_bytes => INT32 + * topics => topic [partitions] + * topic => STRING + * partitions => partition fetch_offset max_bytes + * partition => INT32 + * fetch_offset => INT64 + * max_bytes => INT32 + */ + +/** + * @param {number} replicaId Broker id of the follower + * @param {number} maxWaitTime Maximum time in ms to wait for the response + * @param {number} minBytes Minimum bytes to accumulate in the response. + * @param {number} maxBytes Maximum bytes to accumulate in the response. Note that this is not an absolute maximum, + * if the first message in the first non-empty partition of the fetch is larger than this value, + * the message will still be returned to ensure that progress can be made. + * @param {Array} topics Topics to fetch + * [ + * { + * topic: 'topic-name', + * partitions: [ + * { + * partition: 0, + * fetchOffset: '4124', + * maxBytes: 2048 + * } + * ] + * } + * ] + */ +module.exports = ({ replicaId, maxWaitTime, minBytes, maxBytes, topics }) => ({ + apiKey, + apiVersion: 3, + apiName: 'Fetch', + encode: async () => { + return new Encoder() + .writeInt32(replicaId) + .writeInt32(maxWaitTime) + .writeInt32(minBytes) + .writeInt32(maxBytes) + .writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition, fetchOffset, maxBytes }) => { + return new Encoder() + .writeInt32(partition) + .writeInt64(fetchOffset) + .writeInt32(maxBytes) +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v3/response.js b/node_modules/kafkajs/src/protocol/requests/fetch/v3/response.js new file mode 100644 index 0000000..fe48a27 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v3/response.js @@ -0,0 +1,19 @@ +const { decode, parse } = require('../v1/response') + +/** + * Fetch Response (Version: 3) => throttle_time_ms [responses] + * throttle_time_ms => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition_header record_set + * partition_header => partition error_code high_watermark + * partition => INT32 + * error_code => INT16 + * high_watermark => INT64 + * record_set => RECORDS + */ + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v4/decodeMessages.js b/node_modules/kafkajs/src/protocol/requests/fetch/v4/decodeMessages.js new file mode 100644 index 0000000..abae0f5 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v4/decodeMessages.js @@ -0,0 +1,46 @@ +const Decoder = require('../../../decoder') +const MessageSetDecoder = require('../../../messageSet/decoder') +const RecordBatchDecoder = require('../../../recordBatch/v0/decoder') +const { MAGIC_BYTE } = require('../../../recordBatch/v0') + +// the magic offset is at the same offset for all current message formats, but the 4 bytes +// between the size and the magic is dependent on the version. +const MAGIC_OFFSET = 16 +const RECORD_BATCH_OVERHEAD = 49 + +const decodeMessages = async decoder => { + const messagesSize = decoder.readInt32() + + if (messagesSize <= 0 || !decoder.canReadBytes(messagesSize)) { + return [] + } + + const messagesBuffer = decoder.readBytes(messagesSize) + const messagesDecoder = new Decoder(messagesBuffer) + const magicByte = messagesBuffer.slice(MAGIC_OFFSET).readInt8(0) + + if (magicByte === MAGIC_BYTE) { + const records = [] + + while (messagesDecoder.canReadBytes(RECORD_BATCH_OVERHEAD)) { + try { + const recordBatch = await RecordBatchDecoder(messagesDecoder) + records.push(...recordBatch.records) + } catch (e) { + // The tail of the record batches can have incomplete records + // due to how maxBytes works. See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-FetchAPI + if (e.name === 'KafkaJSPartialMessageError') { + break + } + + throw e + } + } + + return records + } + + return MessageSetDecoder(messagesDecoder, messagesSize) +} + +module.exports = decodeMessages diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v4/request.js b/node_modules/kafkajs/src/protocol/requests/fetch/v4/request.js new file mode 100644 index 0000000..e7d8778 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v4/request.js @@ -0,0 +1,51 @@ +const Encoder = require('../../../encoder') +const { Fetch: apiKey } = require('../../apiKeys') +const ISOLATION_LEVEL = require('../../../isolationLevel') + +/** + * Fetch Request (Version: 4) => replica_id max_wait_time min_bytes max_bytes isolation_level [topics] + * replica_id => INT32 + * max_wait_time => INT32 + * min_bytes => INT32 + * max_bytes => INT32 + * isolation_level => INT8 + * topics => topic [partitions] + * topic => STRING + * partitions => partition fetch_offset max_bytes + * partition => INT32 + * fetch_offset => INT64 + * max_bytes => INT32 + */ + +module.exports = ({ + replicaId, + maxWaitTime, + minBytes, + maxBytes, + topics, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, +}) => ({ + apiKey, + apiVersion: 4, + apiName: 'Fetch', + encode: async () => { + return new Encoder() + .writeInt32(replicaId) + .writeInt32(maxWaitTime) + .writeInt32(minBytes) + .writeInt32(maxBytes) + .writeInt8(isolationLevel) + .writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition, fetchOffset, maxBytes }) => { + return new Encoder() + .writeInt32(partition) + .writeInt64(fetchOffset) + .writeInt32(maxBytes) +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v4/response.js b/node_modules/kafkajs/src/protocol/requests/fetch/v4/response.js new file mode 100644 index 0000000..c6247de --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v4/response.js @@ -0,0 +1,55 @@ +const Decoder = require('../../../decoder') +const { parse: parseV1 } = require('../v1/response') +const decodeMessages = require('./decodeMessages') + +/** + * Fetch Response (Version: 4) => throttle_time_ms [responses] + * throttle_time_ms => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition_header record_set + * partition_header => partition error_code high_watermark last_stable_offset [aborted_transactions] + * partition => INT32 + * error_code => INT16 + * high_watermark => INT64 + * last_stable_offset => INT64 + * aborted_transactions => producer_id first_offset + * producer_id => INT64 + * first_offset => INT64 + * record_set => RECORDS + */ + +const decodeAbortedTransactions = decoder => ({ + producerId: decoder.readInt64().toString(), + firstOffset: decoder.readInt64().toString(), +}) + +const decodePartition = async decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + highWatermark: decoder.readInt64().toString(), + lastStableOffset: decoder.readInt64().toString(), + abortedTransactions: decoder.readArray(decodeAbortedTransactions), + messages: await decodeMessages(decoder), +}) + +const decodeResponse = async decoder => ({ + topicName: decoder.readString(), + partitions: await decoder.readArrayAsync(decodePartition), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const responses = await decoder.readArrayAsync(decodeResponse) + + return { + throttleTime, + responses, + } +} + +module.exports = { + decode, + parse: parseV1, +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v5/request.js b/node_modules/kafkajs/src/protocol/requests/fetch/v5/request.js new file mode 100644 index 0000000..c2df3da --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v5/request.js @@ -0,0 +1,53 @@ +const Encoder = require('../../../encoder') +const { Fetch: apiKey } = require('../../apiKeys') +const ISOLATION_LEVEL = require('../../../isolationLevel') + +/** + * Fetch Request (Version: 5) => replica_id max_wait_time min_bytes max_bytes isolation_level [topics] + * replica_id => INT32 + * max_wait_time => INT32 + * min_bytes => INT32 + * max_bytes => INT32 + * isolation_level => INT8 + * topics => topic [partitions] + * topic => STRING + * partitions => partition fetch_offset log_start_offset partition_max_bytes + * partition => INT32 + * fetch_offset => INT64 + * log_start_offset => INT64 + * partition_max_bytes => INT32 + */ + +module.exports = ({ + replicaId, + maxWaitTime, + minBytes, + maxBytes, + topics, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, +}) => ({ + apiKey, + apiVersion: 5, + apiName: 'Fetch', + encode: async () => { + return new Encoder() + .writeInt32(replicaId) + .writeInt32(maxWaitTime) + .writeInt32(minBytes) + .writeInt32(maxBytes) + .writeInt8(isolationLevel) + .writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition, fetchOffset, logStartOffset = -1, maxBytes }) => { + return new Encoder() + .writeInt32(partition) + .writeInt64(fetchOffset) + .writeInt64(logStartOffset) + .writeInt32(maxBytes) +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v5/response.js b/node_modules/kafkajs/src/protocol/requests/fetch/v5/response.js new file mode 100644 index 0000000..ce63d64 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v5/response.js @@ -0,0 +1,57 @@ +const Decoder = require('../../../decoder') +const { parse: parseV1 } = require('../v1/response') +const decodeMessages = require('../v4/decodeMessages') + +/** + * Fetch Response (Version: 5) => throttle_time_ms [responses] + * throttle_time_ms => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition_header record_set + * partition_header => partition error_code high_watermark last_stable_offset log_start_offset [aborted_transactions] + * partition => INT32 + * error_code => INT16 + * high_watermark => INT64 + * last_stable_offset => INT64 + * log_start_offset => INT64 + * aborted_transactions => producer_id first_offset + * producer_id => INT64 + * first_offset => INT64 + * record_set => RECORDS + */ + +const decodeAbortedTransactions = decoder => ({ + producerId: decoder.readInt64().toString(), + firstOffset: decoder.readInt64().toString(), +}) + +const decodePartition = async decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + highWatermark: decoder.readInt64().toString(), + lastStableOffset: decoder.readInt64().toString(), + lastStartOffset: decoder.readInt64().toString(), + abortedTransactions: decoder.readArray(decodeAbortedTransactions), + messages: await decodeMessages(decoder), +}) + +const decodeResponse = async decoder => ({ + topicName: decoder.readString(), + partitions: await decoder.readArrayAsync(decodePartition), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const responses = await decoder.readArrayAsync(decodeResponse) + + return { + throttleTime, + responses, + } +} + +module.exports = { + decode, + parse: parseV1, +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v6/request.js b/node_modules/kafkajs/src/protocol/requests/fetch/v6/request.js new file mode 100644 index 0000000..e23015c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v6/request.js @@ -0,0 +1,38 @@ +const ISOLATION_LEVEL = require('../../../isolationLevel') +const requestV5 = require('../v5/request') + +/** + * Fetch Request (Version: 6) => replica_id max_wait_time min_bytes max_bytes isolation_level [topics] + * replica_id => INT32 + * max_wait_time => INT32 + * min_bytes => INT32 + * max_bytes => INT32 + * isolation_level => INT8 + * topics => topic [partitions] + * topic => STRING + * partitions => partition fetch_offset log_start_offset partition_max_bytes + * partition => INT32 + * fetch_offset => INT64 + * log_start_offset => INT64 + * partition_max_bytes => INT32 + */ + +module.exports = ({ + replicaId, + maxWaitTime, + minBytes, + maxBytes, + topics, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, +}) => + Object.assign( + requestV5({ + replicaId, + maxWaitTime, + minBytes, + maxBytes, + topics, + isolationLevel, + }), + { apiVersion: 6 } + ) diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v6/response.js b/node_modules/kafkajs/src/protocol/requests/fetch/v6/response.js new file mode 100644 index 0000000..b4d8085 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v6/response.js @@ -0,0 +1,24 @@ +const { decode, parse } = require('../v5/response') + +/** + * Fetch Response (Version: 6) => throttle_time_ms [responses] + * throttle_time_ms => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition_header record_set + * partition_header => partition error_code high_watermark last_stable_offset log_start_offset [aborted_transactions] + * partition => INT32 + * error_code => INT16 + * high_watermark => INT64 + * last_stable_offset => INT64 + * log_start_offset => INT64 + * aborted_transactions => producer_id first_offset + * producer_id => INT64 + * first_offset => INT64 + * record_set => RECORDS + */ + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v7/request.js b/node_modules/kafkajs/src/protocol/requests/fetch/v7/request.js new file mode 100644 index 0000000..3f994a1 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v7/request.js @@ -0,0 +1,73 @@ +const Encoder = require('../../../encoder') +const { Fetch: apiKey } = require('../../apiKeys') +const ISOLATION_LEVEL = require('../../../isolationLevel') + +/** + * Sessions are only used by followers + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-227%3A+Introduce+Incremental+FetchRequests+to+Increase+Partition+Scalability + */ + +/** + * Fetch Request (Version: 7) => replica_id max_wait_time min_bytes max_bytes isolation_level session_id session_epoch [topics] [forgotten_topics_data] + * replica_id => INT32 + * max_wait_time => INT32 + * min_bytes => INT32 + * max_bytes => INT32 + * isolation_level => INT8 + * session_id => INT32 + * session_epoch => INT32 + * topics => topic [partitions] + * topic => STRING + * partitions => partition fetch_offset log_start_offset partition_max_bytes + * partition => INT32 + * fetch_offset => INT64 + * log_start_offset => INT64 + * partition_max_bytes => INT32 + * forgotten_topics_data => topic [partitions] + * topic => STRING + * partitions => INT32 + */ + +module.exports = ({ + replicaId, + maxWaitTime, + minBytes, + maxBytes, + topics, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + sessionId = 0, + sessionEpoch = -1, + forgottenTopics = [], // Topics to remove from the fetch session +}) => ({ + apiKey, + apiVersion: 7, + apiName: 'Fetch', + encode: async () => { + return new Encoder() + .writeInt32(replicaId) + .writeInt32(maxWaitTime) + .writeInt32(minBytes) + .writeInt32(maxBytes) + .writeInt8(isolationLevel) + .writeInt32(sessionId) + .writeInt32(sessionEpoch) + .writeArray(topics.map(encodeTopic)) + .writeArray(forgottenTopics.map(encodeForgottenTopics)) + }, +}) + +const encodeForgottenTopics = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions) +} + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition, fetchOffset, logStartOffset = -1, maxBytes }) => { + return new Encoder() + .writeInt32(partition) + .writeInt64(fetchOffset) + .writeInt64(logStartOffset) + .writeInt32(maxBytes) +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v7/response.js b/node_modules/kafkajs/src/protocol/requests/fetch/v7/response.js new file mode 100644 index 0000000..45c9f6a --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v7/response.js @@ -0,0 +1,63 @@ +const Decoder = require('../../../decoder') +const { parse: parseV1 } = require('../v1/response') +const decodeMessages = require('../v4/decodeMessages') + +/** + * Fetch Response (Version: 7) => throttle_time_ms error_code session_id [responses] + * throttle_time_ms => INT32 + * error_code => INT16 + * session_id => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition_header record_set + * partition_header => partition error_code high_watermark last_stable_offset log_start_offset [aborted_transactions] + * partition => INT32 + * error_code => INT16 + * high_watermark => INT64 + * last_stable_offset => INT64 + * log_start_offset => INT64 + * aborted_transactions => producer_id first_offset + * producer_id => INT64 + * first_offset => INT64 + * record_set => RECORDS + */ + +const decodeAbortedTransactions = decoder => ({ + producerId: decoder.readInt64().toString(), + firstOffset: decoder.readInt64().toString(), +}) + +const decodePartition = async decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + highWatermark: decoder.readInt64().toString(), + lastStableOffset: decoder.readInt64().toString(), + lastStartOffset: decoder.readInt64().toString(), + abortedTransactions: decoder.readArray(decodeAbortedTransactions), + messages: await decodeMessages(decoder), +}) + +const decodeResponse = async decoder => ({ + topicName: decoder.readString(), + partitions: await decoder.readArrayAsync(decodePartition), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + const sessionId = decoder.readInt32() + const responses = await decoder.readArrayAsync(decodeResponse) + + return { + throttleTime, + errorCode, + sessionId, + responses, + } +} + +module.exports = { + decode, + parse: parseV1, +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v8/request.js b/node_modules/kafkajs/src/protocol/requests/fetch/v8/request.js new file mode 100644 index 0000000..420df24 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v8/request.js @@ -0,0 +1,54 @@ +const ISOLATION_LEVEL = require('../../../isolationLevel') +const requestV7 = require('../v7/request') + +/** + * Quota violation brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + */ + +/** + * Fetch Request (Version: 8) => replica_id max_wait_time min_bytes max_bytes isolation_level session_id session_epoch [topics] [forgotten_topics_data] + * replica_id => INT32 + * max_wait_time => INT32 + * min_bytes => INT32 + * max_bytes => INT32 + * isolation_level => INT8 + * session_id => INT32 + * session_epoch => INT32 + * topics => topic [partitions] + * topic => STRING + * partitions => partition fetch_offset log_start_offset partition_max_bytes + * partition => INT32 + * fetch_offset => INT64 + * log_start_offset => INT64 + * partition_max_bytes => INT32 + * forgotten_topics_data => topic [partitions] + * topic => STRING + * partitions => INT32 + */ + +module.exports = ({ + replicaId, + maxWaitTime, + minBytes, + maxBytes, + topics, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + sessionId = 0, + sessionEpoch = -1, + forgottenTopics = [], // Topics to remove from the fetch session +}) => + Object.assign( + requestV7({ + replicaId, + maxWaitTime, + minBytes, + maxBytes, + topics, + isolationLevel, + sessionId, + sessionEpoch, + forgottenTopics, + }), + { apiVersion: 8 } + ) diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v8/response.js b/node_modules/kafkajs/src/protocol/requests/fetch/v8/response.js new file mode 100644 index 0000000..dc61f8d --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v8/response.js @@ -0,0 +1,67 @@ +const Decoder = require('../../../decoder') +const { parse: parseV1 } = require('../v1/response') +const decodeMessages = require('../v4/decodeMessages') + +/** + * Fetch Response (Version: 8) => throttle_time_ms error_code session_id [responses] + * throttle_time_ms => INT32 + * error_code => INT16 + * session_id => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition_header record_set + * partition_header => partition error_code high_watermark last_stable_offset log_start_offset [aborted_transactions] + * partition => INT32 + * error_code => INT16 + * high_watermark => INT64 + * last_stable_offset => INT64 + * log_start_offset => INT64 + * aborted_transactions => producer_id first_offset + * producer_id => INT64 + * first_offset => INT64 + * record_set => RECORDS + */ + +const decodeAbortedTransactions = decoder => ({ + producerId: decoder.readInt64().toString(), + firstOffset: decoder.readInt64().toString(), +}) + +const decodePartition = async decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + highWatermark: decoder.readInt64().toString(), + lastStableOffset: decoder.readInt64().toString(), + lastStartOffset: decoder.readInt64().toString(), + abortedTransactions: decoder.readArray(decodeAbortedTransactions), + messages: await decodeMessages(decoder), +}) + +const decodeResponse = async decoder => ({ + topicName: decoder.readString(), + partitions: await decoder.readArrayAsync(decodePartition), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const clientSideThrottleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + const sessionId = decoder.readInt32() + const responses = await decoder.readArrayAsync(decodeResponse) + + // Report a `throttleTime` of 0: The broker will not have throttled + // this request, but if the `clientSideThrottleTime` is >0 then it + // expects us to do that -- and it will ignore requests. + return { + throttleTime: 0, + clientSideThrottleTime, + errorCode, + sessionId, + responses, + } +} + +module.exports = { + decode, + parse: parseV1, +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v9/request.js b/node_modules/kafkajs/src/protocol/requests/fetch/v9/request.js new file mode 100644 index 0000000..4ebdc59 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v9/request.js @@ -0,0 +1,81 @@ +const Encoder = require('../../../encoder') +const { Fetch: apiKey } = require('../../apiKeys') +const ISOLATION_LEVEL = require('../../../isolationLevel') + +/** + * Allow fetchers to detect and handle log truncation + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-320%3A+Allow+fetchers+to+detect+and+handle+log+truncation + */ + +/** + * Fetch Request (Version: 9) => replica_id max_wait_time min_bytes max_bytes isolation_level session_id session_epoch [topics] [forgotten_topics_data] + * replica_id => INT32 + * max_wait_time => INT32 + * min_bytes => INT32 + * max_bytes => INT32 + * isolation_level => INT8 + * session_id => INT32 + * session_epoch => INT32 + * topics => topic [partitions] + * topic => STRING + * partitions => partition current_leader_epoch fetch_offset log_start_offset partition_max_bytes + * partition => INT32 + * current_leader_epoch => INT32 + * fetch_offset => INT64 + * log_start_offset => INT64 + * partition_max_bytes => INT32 + * forgotten_topics_data => topic [partitions] + * topic => STRING + * partitions => INT32 + */ + +module.exports = ({ + replicaId, + maxWaitTime, + minBytes, + maxBytes, + topics, + isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, + sessionId = 0, + sessionEpoch = -1, + forgottenTopics = [], // Topics to remove from the fetch session +}) => ({ + apiKey, + apiVersion: 9, + apiName: 'Fetch', + encode: async () => { + return new Encoder() + .writeInt32(replicaId) + .writeInt32(maxWaitTime) + .writeInt32(minBytes) + .writeInt32(maxBytes) + .writeInt8(isolationLevel) + .writeInt32(sessionId) + .writeInt32(sessionEpoch) + .writeArray(topics.map(encodeTopic)) + .writeArray(forgottenTopics.map(encodeForgottenTopics)) + }, +}) + +const encodeForgottenTopics = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions) +} + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ + partition, + currentLeaderEpoch = -1, + fetchOffset, + logStartOffset = -1, + maxBytes, +}) => { + return new Encoder() + .writeInt32(partition) + .writeInt32(currentLeaderEpoch) + .writeInt64(fetchOffset) + .writeInt64(logStartOffset) + .writeInt32(maxBytes) +} diff --git a/node_modules/kafkajs/src/protocol/requests/fetch/v9/response.js b/node_modules/kafkajs/src/protocol/requests/fetch/v9/response.js new file mode 100644 index 0000000..0e82b0e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/fetch/v9/response.js @@ -0,0 +1,26 @@ +const { decode, parse } = require('../v8/response') + +/** + * Fetch Response (Version: 9) => throttle_time_ms error_code session_id [responses] + * throttle_time_ms => INT32 + * error_code => INT16 + * session_id => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition_header record_set + * partition_header => partition error_code high_watermark last_stable_offset log_start_offset [aborted_transactions] + * partition => INT32 + * error_code => INT16 + * high_watermark => INT64 + * last_stable_offset => INT64 + * log_start_offset => INT64 + * aborted_transactions => producer_id first_offset + * producer_id => INT64 + * first_offset => INT64 + * record_set => RECORDS + */ + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/findCoordinator/index.js b/node_modules/kafkajs/src/protocol/requests/findCoordinator/index.js new file mode 100644 index 0000000..4c486de --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/findCoordinator/index.js @@ -0,0 +1,24 @@ +const COORDINATOR_TYPES = require('../../coordinatorTypes') + +const versions = { + 0: ({ groupId }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ groupId }), response } + }, + 1: ({ groupId, coordinatorType = COORDINATOR_TYPES.GROUP }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ coordinatorKey: groupId, coordinatorType }), response } + }, + 2: ({ groupId, coordinatorType = COORDINATOR_TYPES.GROUP }) => { + const request = require('./v2/request') + const response = require('./v2/response') + return { request: request({ coordinatorKey: groupId, coordinatorType }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/findCoordinator/v0/request.js b/node_modules/kafkajs/src/protocol/requests/findCoordinator/v0/request.js new file mode 100644 index 0000000..7fbf421 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/findCoordinator/v0/request.js @@ -0,0 +1,16 @@ +const Encoder = require('../../../encoder') +const { GroupCoordinator: apiKey } = require('../../apiKeys') + +/** + * FindCoordinator Request (Version: 0) => group_id + * group_id => STRING + */ + +module.exports = ({ groupId }) => ({ + apiKey, + apiVersion: 0, + apiName: 'GroupCoordinator', + encode: async () => { + return new Encoder().writeString(groupId) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/findCoordinator/v0/response.js b/node_modules/kafkajs/src/protocol/requests/findCoordinator/v0/response.js new file mode 100644 index 0000000..8eb97bc --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/findCoordinator/v0/response.js @@ -0,0 +1,42 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode, failIfVersionNotSupported } = require('../../../error') + +/** + * FindCoordinator Response (Version: 0) => error_code coordinator + * error_code => INT16 + * coordinator => node_id host port + * node_id => INT32 + * host => STRING + * port => INT32 + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + const coordinator = { + nodeId: decoder.readInt32(), + host: decoder.readString(), + port: decoder.readInt32(), + } + + return { + errorCode, + coordinator, + } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/findCoordinator/v1/request.js b/node_modules/kafkajs/src/protocol/requests/findCoordinator/v1/request.js new file mode 100644 index 0000000..c1340b8 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/findCoordinator/v1/request.js @@ -0,0 +1,17 @@ +const Encoder = require('../../../encoder') +const { GroupCoordinator: apiKey } = require('../../apiKeys') + +/** + * FindCoordinator Request (Version: 1) => coordinator_key coordinator_type + * coordinator_key => STRING + * coordinator_type => INT8 + */ + +module.exports = ({ coordinatorKey, coordinatorType }) => ({ + apiKey, + apiVersion: 1, + apiName: 'GroupCoordinator', + encode: async () => { + return new Encoder().writeString(coordinatorKey).writeInt8(coordinatorType) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/findCoordinator/v1/response.js b/node_modules/kafkajs/src/protocol/requests/findCoordinator/v1/response.js new file mode 100644 index 0000000..be72022 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/findCoordinator/v1/response.js @@ -0,0 +1,48 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode, failIfVersionNotSupported } = require('../../../error') + +/** + * FindCoordinator Response (Version: 1) => throttle_time_ms error_code error_message coordinator + * throttle_time_ms => INT32 + * error_code => INT16 + * error_message => NULLABLE_STRING + * coordinator => node_id host port + * node_id => INT32 + * host => STRING + * port => INT32 + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + const errorMessage = decoder.readString() + const coordinator = { + nodeId: decoder.readInt32(), + host: decoder.readString(), + port: decoder.readInt32(), + } + + return { + throttleTime, + errorCode, + errorMessage, + coordinator, + } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/findCoordinator/v2/request.js b/node_modules/kafkajs/src/protocol/requests/findCoordinator/v2/request.js new file mode 100644 index 0000000..ebe22a9 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/findCoordinator/v2/request.js @@ -0,0 +1,10 @@ +const requestV1 = require('../v1/request') + +/** + * FindCoordinator Request (Version: 2) => coordinator_key coordinator_type + * coordinator_key => STRING + * coordinator_type => INT8 + */ + +module.exports = ({ coordinatorKey, coordinatorType }) => + Object.assign(requestV1({ coordinatorKey, coordinatorType }), { apiVersion: 2 }) diff --git a/node_modules/kafkajs/src/protocol/requests/findCoordinator/v2/response.js b/node_modules/kafkajs/src/protocol/requests/findCoordinator/v2/response.js new file mode 100644 index 0000000..f31d67b --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/findCoordinator/v2/response.js @@ -0,0 +1,30 @@ +const { parse, decode: decodeV1 } = require('../v1/response') + +/** + * Starting in version 2, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * FindCoordinator Response (Version: 1) => throttle_time_ms error_code error_message coordinator + * throttle_time_ms => INT32 + * error_code => INT16 + * error_message => NULLABLE_STRING + * coordinator => node_id host port + * node_id => INT32 + * host => STRING + * port => INT32 + */ + +const decode = async rawData => { + const decoded = await decodeV1(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/heartbeat/index.js b/node_modules/kafkajs/src/protocol/requests/heartbeat/index.js new file mode 100644 index 0000000..3c090ad --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/heartbeat/index.js @@ -0,0 +1,39 @@ +const versions = { + 0: ({ groupId, groupGenerationId, memberId }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { + request: request({ groupId, groupGenerationId, memberId }), + response, + } + }, + 1: ({ groupId, groupGenerationId, memberId }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { + request: request({ groupId, groupGenerationId, memberId }), + response, + } + }, + 2: ({ groupId, groupGenerationId, memberId }) => { + const request = require('./v2/request') + const response = require('./v2/response') + return { + request: request({ groupId, groupGenerationId, memberId }), + response, + } + }, + 3: ({ groupId, groupGenerationId, memberId, groupInstanceId }) => { + const request = require('./v3/request') + const response = require('./v3/response') + return { + request: request({ groupId, groupGenerationId, memberId, groupInstanceId }), + response, + } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/heartbeat/v0/request.js b/node_modules/kafkajs/src/protocol/requests/heartbeat/v0/request.js new file mode 100644 index 0000000..137dd27 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/heartbeat/v0/request.js @@ -0,0 +1,21 @@ +const Encoder = require('../../../encoder') +const { Heartbeat: apiKey } = require('../../apiKeys') + +/** + * Heartbeat Request (Version: 0) => group_id group_generation_id member_id + * group_id => STRING + * group_generation_id => INT32 + * member_id => STRING + */ + +module.exports = ({ groupId, groupGenerationId, memberId }) => ({ + apiKey, + apiVersion: 0, + apiName: 'Heartbeat', + encode: async () => { + return new Encoder() + .writeString(groupId) + .writeInt32(groupGenerationId) + .writeString(memberId) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/heartbeat/v0/response.js b/node_modules/kafkajs/src/protocol/requests/heartbeat/v0/response.js new file mode 100644 index 0000000..cf7333b --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/heartbeat/v0/response.js @@ -0,0 +1,29 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode, failIfVersionNotSupported } = require('../../../error') + +/** + * Heartbeat Response (Version: 0) => error_code + * error_code => INT16 + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { errorCode } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/heartbeat/v1/request.js b/node_modules/kafkajs/src/protocol/requests/heartbeat/v1/request.js new file mode 100644 index 0000000..c11546b --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/heartbeat/v1/request.js @@ -0,0 +1,11 @@ +const requestV0 = require('../v0/request') + +/** + * Heartbeat Request (Version: 1) => group_id generation_id member_id + * group_id => STRING + * generation_id => INT32 + * member_id => STRING + */ + +module.exports = ({ groupId, groupGenerationId, memberId }) => + Object.assign(requestV0({ groupId, groupGenerationId, memberId }), { apiVersion: 1 }) diff --git a/node_modules/kafkajs/src/protocol/requests/heartbeat/v1/response.js b/node_modules/kafkajs/src/protocol/requests/heartbeat/v1/response.js new file mode 100644 index 0000000..334de58 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/heartbeat/v1/response.js @@ -0,0 +1,24 @@ +const Decoder = require('../../../decoder') +const { failIfVersionNotSupported } = require('../../../error') +const { parse: parseV0 } = require('../v0/response') + +/** + * Heartbeat Response (Version: 1) => throttle_time_ms error_code + * throttle_time_ms => INT32 + * error_code => INT16 + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { throttleTime, errorCode } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/heartbeat/v2/request.js b/node_modules/kafkajs/src/protocol/requests/heartbeat/v2/request.js new file mode 100644 index 0000000..75118da --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/heartbeat/v2/request.js @@ -0,0 +1,11 @@ +const requestV1 = require('../v1/request') + +/** + * Heartbeat Request (Version: 2) => group_id generation_id member_id + * group_id => STRING + * generation_id => INT32 + * member_id => STRING + */ + +module.exports = ({ groupId, groupGenerationId, memberId }) => + Object.assign(requestV1({ groupId, groupGenerationId, memberId }), { apiVersion: 2 }) diff --git a/node_modules/kafkajs/src/protocol/requests/heartbeat/v2/response.js b/node_modules/kafkajs/src/protocol/requests/heartbeat/v2/response.js new file mode 100644 index 0000000..86c6f20 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/heartbeat/v2/response.js @@ -0,0 +1,24 @@ +const { parse, decode: decodeV1 } = require('../v1/response') + +/** + * In version 2 on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * Heartbeat Response (Version: 2) => throttle_time_ms error_code + * throttle_time_ms => INT32 + * error_code => INT16 + */ +const decode = async rawData => { + const decoded = await decodeV1(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/heartbeat/v3/request.js b/node_modules/kafkajs/src/protocol/requests/heartbeat/v3/request.js new file mode 100644 index 0000000..00b86ae --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/heartbeat/v3/request.js @@ -0,0 +1,26 @@ +const Encoder = require('../../../encoder') +const { Heartbeat: apiKey } = require('../../apiKeys') + +/** + * Version 3 adds group_instance_id to indicate member identity across restarts. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances + * + * Heartbeat Request (Version: 3) => group_id generation_id member_id group_instance_id + * group_id => STRING + * generation_id => INT32 + * member_id => STRING + * group_instance_id => NULLABLE_STRING + */ + +module.exports = ({ groupId, groupGenerationId, memberId, groupInstanceId }) => ({ + apiKey, + apiVersion: 3, + apiName: 'Heartbeat', + encode: async () => { + return new Encoder() + .writeString(groupId) + .writeInt32(groupGenerationId) + .writeString(memberId) + .writeString(groupInstanceId) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/heartbeat/v3/response.js b/node_modules/kafkajs/src/protocol/requests/heartbeat/v3/response.js new file mode 100644 index 0000000..c4a5ec5 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/heartbeat/v3/response.js @@ -0,0 +1,11 @@ +const { parse, decode } = require('../v2/response') + +/** + * Heartbeat Response (Version: 3) => throttle_time_ms error_code + * throttle_time_ms => INT32 + * error_code => INT16 + */ +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/index.js b/node_modules/kafkajs/src/protocol/requests/index.js new file mode 100644 index 0000000..96137bc --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/index.js @@ -0,0 +1,106 @@ +const apiKeys = require('./apiKeys') +const { KafkaJSServerDoesNotSupportApiKey, KafkaJSNotImplemented } = require('../../errors') + +/** + * @typedef {(options?: Object) => { request: any, response: any, logResponseErrors?: boolean }} Request + */ + +/** + * @typedef {Object} RequestDefinitions + * @property {string[]} versions + * @property {({ version: number }) => Request} protocol + */ + +/** + * @typedef {(apiKey: number, definitions: RequestDefinitions) => Request} Lookup + */ + +/** @type {RequestDefinitions} */ +const noImplementedRequestDefinitions = { + versions: [], + protocol: () => { + throw new KafkaJSNotImplemented() + }, +} + +/** + * @type {{[apiName: string]: RequestDefinitions}} + */ +const requests = { + Produce: require('./produce'), + Fetch: require('./fetch'), + ListOffsets: require('./listOffsets'), + Metadata: require('./metadata'), + LeaderAndIsr: noImplementedRequestDefinitions, + StopReplica: noImplementedRequestDefinitions, + UpdateMetadata: noImplementedRequestDefinitions, + ControlledShutdown: noImplementedRequestDefinitions, + OffsetCommit: require('./offsetCommit'), + OffsetFetch: require('./offsetFetch'), + GroupCoordinator: require('./findCoordinator'), + JoinGroup: require('./joinGroup'), + Heartbeat: require('./heartbeat'), + LeaveGroup: require('./leaveGroup'), + SyncGroup: require('./syncGroup'), + DescribeGroups: require('./describeGroups'), + ListGroups: require('./listGroups'), + SaslHandshake: require('./saslHandshake'), + ApiVersions: require('./apiVersions'), + CreateTopics: require('./createTopics'), + DeleteTopics: require('./deleteTopics'), + DeleteRecords: require('./deleteRecords'), + InitProducerId: require('./initProducerId'), + OffsetForLeaderEpoch: noImplementedRequestDefinitions, + AddPartitionsToTxn: require('./addPartitionsToTxn'), + AddOffsetsToTxn: require('./addOffsetsToTxn'), + EndTxn: require('./endTxn'), + WriteTxnMarkers: noImplementedRequestDefinitions, + TxnOffsetCommit: require('./txnOffsetCommit'), + DescribeAcls: require('./describeAcls'), + CreateAcls: require('./createAcls'), + DeleteAcls: require('./deleteAcls'), + DescribeConfigs: require('./describeConfigs'), + AlterConfigs: require('./alterConfigs'), + AlterReplicaLogDirs: noImplementedRequestDefinitions, + DescribeLogDirs: noImplementedRequestDefinitions, + SaslAuthenticate: require('./saslAuthenticate'), + CreatePartitions: require('./createPartitions'), + CreateDelegationToken: noImplementedRequestDefinitions, + RenewDelegationToken: noImplementedRequestDefinitions, + ExpireDelegationToken: noImplementedRequestDefinitions, + DescribeDelegationToken: noImplementedRequestDefinitions, + DeleteGroups: require('./deleteGroups'), + ElectLeaders: noImplementedRequestDefinitions, + IncrementalAlterConfigs: noImplementedRequestDefinitions, + AlterPartitionReassignments: require('./alterPartitionReassignments'), + ListPartitionReassignments: require('./listPartitionReassignments'), +} + +const names = Object.keys(apiKeys) +const keys = Object.values(apiKeys) +const findApiName = apiKey => names[keys.indexOf(apiKey)] + +/** + * @param {import("../../../types").ApiVersions} versions + * @returns {Lookup} + */ +const lookup = versions => (apiKey, definition) => { + const version = versions[apiKey] + const availableVersions = definition.versions.map(Number) + const bestImplementedVersion = Math.max(...availableVersions) + + if (!version || version.maxVersion == null) { + throw new KafkaJSServerDoesNotSupportApiKey( + `The Kafka server does not support the requested API version`, + { apiKey, apiName: findApiName(apiKey) } + ) + } + + const bestSupportedVersion = Math.min(bestImplementedVersion, version.maxVersion) + return definition.protocol({ version: bestSupportedVersion }) +} + +module.exports = { + requests, + lookup, +} diff --git a/node_modules/kafkajs/src/protocol/requests/initProducerId/index.js b/node_modules/kafkajs/src/protocol/requests/initProducerId/index.js new file mode 100644 index 0000000..907cc01 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/initProducerId/index.js @@ -0,0 +1,17 @@ +const versions = { + 0: ({ transactionalId, transactionTimeout = 5000 }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ transactionalId, transactionTimeout }), response } + }, + 1: ({ transactionalId, transactionTimeout = 5000 }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ transactionalId, transactionTimeout }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/initProducerId/v0/request.js b/node_modules/kafkajs/src/protocol/requests/initProducerId/v0/request.js new file mode 100644 index 0000000..5f16b36 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/initProducerId/v0/request.js @@ -0,0 +1,17 @@ +const Encoder = require('../../../encoder') +const { InitProducerId: apiKey } = require('../../apiKeys') + +/** + * InitProducerId Request (Version: 0) => transactional_id transaction_timeout_ms + * transactional_id => NULLABLE_STRING + * transaction_timeout_ms => INT32 + */ + +module.exports = ({ transactionalId, transactionTimeout }) => ({ + apiKey, + apiVersion: 0, + apiName: 'InitProducerId', + encode: async () => { + return new Encoder().writeString(transactionalId).writeInt32(transactionTimeout) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/initProducerId/v0/response.js b/node_modules/kafkajs/src/protocol/requests/initProducerId/v0/response.js new file mode 100644 index 0000000..92fdac8 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/initProducerId/v0/response.js @@ -0,0 +1,37 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode, failIfVersionNotSupported } = require('../../../error') + +/** + * InitProducerId Response (Version: 0) => throttle_time_ms error_code producer_id producer_epoch + * throttle_time_ms => INT32 + * error_code => INT16 + * producer_id => INT64 + * producer_epoch => INT16 + */ +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { + throttleTime, + errorCode, + producerId: decoder.readInt64().toString(), + producerEpoch: decoder.readInt16(), + } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/initProducerId/v1/request.js b/node_modules/kafkajs/src/protocol/requests/initProducerId/v1/request.js new file mode 100644 index 0000000..833ec96 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/initProducerId/v1/request.js @@ -0,0 +1,10 @@ +const requestV0 = require('../v0/request') + +/** + * InitProducerId Request (Version: 1) => transactional_id transaction_timeout_ms + * transactional_id => NULLABLE_STRING + * transaction_timeout_ms => INT32 + */ + +module.exports = ({ transactionalId, transactionTimeout }) => + Object.assign(requestV0({ transactionalId, transactionTimeout }), { apiVersion: 1 }) diff --git a/node_modules/kafkajs/src/protocol/requests/initProducerId/v1/response.js b/node_modules/kafkajs/src/protocol/requests/initProducerId/v1/response.js new file mode 100644 index 0000000..41fe0ec --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/initProducerId/v1/response.js @@ -0,0 +1,27 @@ +const { parse, decode: decodeV0 } = require('../v0/response') + +/** + * Starting in version 1, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * InitProducerId Response (Version: 0) => throttle_time_ms error_code producer_id producer_epoch + * throttle_time_ms => INT32 + * error_code => INT16 + * producer_id => INT64 + * producer_epoch => INT16 + */ + +const decode = async rawData => { + const decoded = await decodeV0(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/joinGroup/index.js b/node_modules/kafkajs/src/protocol/requests/joinGroup/index.js new file mode 100644 index 0000000..fc6380a --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/joinGroup/index.js @@ -0,0 +1,135 @@ +const NETWORK_DELAY = 5000 + +/** + * @see https://github.com/apache/kafka/pull/5203 + * The JOIN_GROUP request may block up to sessionTimeout (or rebalanceTimeout in JoinGroupV1), + * so we should override the requestTimeout to be a bit more than the sessionTimeout + * NOTE: the sessionTimeout can be configured as Number.MAX_SAFE_INTEGER and overflow when + * increased, so we have to check for potential overflows + **/ +const requestTimeout = ({ rebalanceTimeout, sessionTimeout }) => { + const timeout = rebalanceTimeout || sessionTimeout + return Number.isSafeInteger(timeout + NETWORK_DELAY) ? timeout + NETWORK_DELAY : timeout +} + +const logResponseError = memberId => memberId != null && memberId !== '' + +const versions = { + 0: ({ groupId, sessionTimeout, memberId, protocolType, groupProtocols }) => { + const request = require('./v0/request') + const response = require('./v0/response') + + return { + request: request({ + groupId, + sessionTimeout, + memberId, + protocolType, + groupProtocols, + }), + response, + requestTimeout: requestTimeout({ rebalanceTimeout: null, sessionTimeout }), + } + }, + 1: ({ groupId, sessionTimeout, rebalanceTimeout, memberId, protocolType, groupProtocols }) => { + const request = require('./v1/request') + const response = require('./v1/response') + + return { + request: request({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + protocolType, + groupProtocols, + }), + response, + requestTimeout: requestTimeout({ rebalanceTimeout, sessionTimeout }), + } + }, + 2: ({ groupId, sessionTimeout, rebalanceTimeout, memberId, protocolType, groupProtocols }) => { + const request = require('./v2/request') + const response = require('./v2/response') + + return { + request: request({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + protocolType, + groupProtocols, + }), + response, + requestTimeout: requestTimeout({ rebalanceTimeout, sessionTimeout }), + } + }, + 3: ({ groupId, sessionTimeout, rebalanceTimeout, memberId, protocolType, groupProtocols }) => { + const request = require('./v3/request') + const response = require('./v3/response') + + return { + request: request({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + protocolType, + groupProtocols, + }), + response, + requestTimeout: requestTimeout({ rebalanceTimeout, sessionTimeout }), + } + }, + 4: ({ groupId, sessionTimeout, rebalanceTimeout, memberId, protocolType, groupProtocols }) => { + const request = require('./v4/request') + const response = require('./v4/response') + + return { + request: request({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + protocolType, + groupProtocols, + }), + response, + requestTimeout: requestTimeout({ rebalanceTimeout, sessionTimeout }), + logResponseError: logResponseError(memberId), + } + }, + 5: ({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + groupInstanceId, + protocolType, + groupProtocols, + }) => { + const request = require('./v5/request') + const response = require('./v5/response') + + return { + request: request({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + groupInstanceId, + protocolType, + groupProtocols, + }), + response, + requestTimeout: requestTimeout({ rebalanceTimeout, sessionTimeout }), + logResponseError: logResponseError(memberId), + } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/joinGroup/v0/request.js b/node_modules/kafkajs/src/protocol/requests/joinGroup/v0/request.js new file mode 100644 index 0000000..e187c21 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/joinGroup/v0/request.js @@ -0,0 +1,31 @@ +const Encoder = require('../../../encoder') +const { JoinGroup: apiKey } = require('../../apiKeys') + +/** + * JoinGroup Request (Version: 0) => group_id session_timeout member_id protocol_type [group_protocols] + * group_id => STRING + * session_timeout => INT32 + * member_id => STRING + * protocol_type => STRING + * group_protocols => protocol_name protocol_metadata + * protocol_name => STRING + * protocol_metadata => BYTES + */ + +module.exports = ({ groupId, sessionTimeout, memberId, protocolType, groupProtocols }) => ({ + apiKey, + apiVersion: 0, + apiName: 'JoinGroup', + encode: async () => { + return new Encoder() + .writeString(groupId) + .writeInt32(sessionTimeout) + .writeString(memberId) + .writeString(protocolType) + .writeArray(groupProtocols.map(encodeGroupProtocols)) + }, +}) + +const encodeGroupProtocols = ({ name, metadata = Buffer.alloc(0) }) => { + return new Encoder().writeString(name).writeBytes(metadata) +} diff --git a/node_modules/kafkajs/src/protocol/requests/joinGroup/v0/response.js b/node_modules/kafkajs/src/protocol/requests/joinGroup/v0/response.js new file mode 100644 index 0000000..827e554 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/joinGroup/v0/response.js @@ -0,0 +1,46 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode, failIfVersionNotSupported } = require('../../../error') + +/** + * JoinGroup Response (Version: 0) => error_code generation_id group_protocol leader_id member_id [members] + * error_code => INT16 + * generation_id => INT32 + * group_protocol => STRING + * leader_id => STRING + * member_id => STRING + * members => member_id member_metadata + * member_id => STRING + * member_metadata => BYTES + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { + errorCode, + generationId: decoder.readInt32(), + groupProtocol: decoder.readString(), + leaderId: decoder.readString(), + memberId: decoder.readString(), + members: decoder.readArray(decoder => ({ + memberId: decoder.readString(), + memberMetadata: decoder.readBytes(), + })), + } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/joinGroup/v1/request.js b/node_modules/kafkajs/src/protocol/requests/joinGroup/v1/request.js new file mode 100644 index 0000000..01ce390 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/joinGroup/v1/request.js @@ -0,0 +1,40 @@ +const Encoder = require('../../../encoder') +const { JoinGroup: apiKey } = require('../../apiKeys') + +/** + * JoinGroup Request (Version: 1) => group_id session_timeout rebalance_timeout member_id protocol_type [group_protocols] + * group_id => STRING + * session_timeout => INT32 + * rebalance_timeout => INT32 + * member_id => STRING + * protocol_type => STRING + * group_protocols => protocol_name protocol_metadata + * protocol_name => STRING + * protocol_metadata => BYTES + */ + +module.exports = ({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + protocolType, + groupProtocols, +}) => ({ + apiKey, + apiVersion: 1, + apiName: 'JoinGroup', + encode: async () => { + return new Encoder() + .writeString(groupId) + .writeInt32(sessionTimeout) + .writeInt32(rebalanceTimeout) + .writeString(memberId) + .writeString(protocolType) + .writeArray(groupProtocols.map(encodeGroupProtocols)) + }, +}) + +const encodeGroupProtocols = ({ name, metadata = Buffer.alloc(0) }) => { + return new Encoder().writeString(name).writeBytes(metadata) +} diff --git a/node_modules/kafkajs/src/protocol/requests/joinGroup/v1/response.js b/node_modules/kafkajs/src/protocol/requests/joinGroup/v1/response.js new file mode 100644 index 0000000..538c21f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/joinGroup/v1/response.js @@ -0,0 +1,18 @@ +const { parse, decode } = require('../v0/response') + +/** + * JoinGroup Response (Version: 1) => error_code generation_id group_protocol leader_id member_id [members] + * error_code => INT16 + * generation_id => INT32 + * group_protocol => STRING + * leader_id => STRING + * member_id => STRING + * members => member_id member_metadata + * member_id => STRING + * member_metadata => BYTES + */ + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/joinGroup/v2/request.js b/node_modules/kafkajs/src/protocol/requests/joinGroup/v2/request.js new file mode 100644 index 0000000..6e78524 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/joinGroup/v2/request.js @@ -0,0 +1,33 @@ +const requestV1 = require('../v1/request') + +/** + * JoinGroup Request (Version: 2) => group_id session_timeout rebalance_timeout member_id protocol_type [group_protocols] + * group_id => STRING + * session_timeout => INT32 + * rebalance_timeout => INT32 + * member_id => STRING + * protocol_type => STRING + * group_protocols => protocol_name protocol_metadata + * protocol_name => STRING + * protocol_metadata => BYTES + */ + +module.exports = ({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + protocolType, + groupProtocols, +}) => + Object.assign( + requestV1({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + protocolType, + groupProtocols, + }), + { apiVersion: 2 } + ) diff --git a/node_modules/kafkajs/src/protocol/requests/joinGroup/v2/response.js b/node_modules/kafkajs/src/protocol/requests/joinGroup/v2/response.js new file mode 100644 index 0000000..b6312c5 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/joinGroup/v2/response.js @@ -0,0 +1,42 @@ +const Decoder = require('../../../decoder') +const { failIfVersionNotSupported } = require('../../../error') +const { parse: parseV0 } = require('../v0/response') + +/** + * JoinGroup Response (Version: 2) => throttle_time_ms error_code generation_id group_protocol leader_id member_id [members] + * throttle_time_ms => INT32 + * error_code => INT16 + * generation_id => INT32 + * group_protocol => STRING + * leader_id => STRING + * member_id => STRING + * members => member_id member_metadata + * member_id => STRING + * member_metadata => BYTES + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { + throttleTime, + errorCode, + generationId: decoder.readInt32(), + groupProtocol: decoder.readString(), + leaderId: decoder.readString(), + memberId: decoder.readString(), + members: decoder.readArray(decoder => ({ + memberId: decoder.readString(), + memberMetadata: decoder.readBytes(), + })), + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/joinGroup/v3/request.js b/node_modules/kafkajs/src/protocol/requests/joinGroup/v3/request.js new file mode 100644 index 0000000..6cd7354 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/joinGroup/v3/request.js @@ -0,0 +1,33 @@ +const requestV2 = require('../v2/request') + +/** + * JoinGroup Request (Version: 3) => group_id session_timeout rebalance_timeout member_id protocol_type [group_protocols] + * group_id => STRING + * session_timeout => INT32 + * rebalance_timeout => INT32 + * member_id => STRING + * protocol_type => STRING + * group_protocols => protocol_name protocol_metadata + * protocol_name => STRING + * protocol_metadata => BYTES + */ + +module.exports = ({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + protocolType, + groupProtocols, +}) => + Object.assign( + requestV2({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + protocolType, + groupProtocols, + }), + { apiVersion: 3 } + ) diff --git a/node_modules/kafkajs/src/protocol/requests/joinGroup/v3/response.js b/node_modules/kafkajs/src/protocol/requests/joinGroup/v3/response.js new file mode 100644 index 0000000..0a613b7 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/joinGroup/v3/response.js @@ -0,0 +1,32 @@ +const { parse, decode: decodeV2 } = require('../v2/response') + +/** + * Starting in version 3, on quota violation, brokers send out responses + * before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * JoinGroup Response (Version: 3) => throttle_time_ms error_code generation_id group_protocol leader_id member_id [members] + * throttle_time_ms => INT32 + * error_code => INT16 + * generation_id => INT32 + * group_protocol => STRING + * leader_id => STRING + * member_id => STRING + * members => member_id member_metadata + * member_id => STRING + * member_metadata => BYTES + */ +const decode = async rawData => { + const decoded = await decodeV2(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/joinGroup/v4/request.js b/node_modules/kafkajs/src/protocol/requests/joinGroup/v4/request.js new file mode 100644 index 0000000..ff21692 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/joinGroup/v4/request.js @@ -0,0 +1,36 @@ +const requestV3 = require('../v3/request') + +/** + * Starting in version 4, the client needs to issue a second request to join group + * with assigned id. + * + * JoinGroup Request (Version: 4) => group_id session_timeout rebalance_timeout member_id protocol_type [group_protocols] + * group_id => STRING + * session_timeout => INT32 + * rebalance_timeout => INT32 + * member_id => STRING + * protocol_type => STRING + * group_protocols => protocol_name protocol_metadata + * protocol_name => STRING + * protocol_metadata => BYTES + */ + +module.exports = ({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + protocolType, + groupProtocols, +}) => + Object.assign( + requestV3({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + protocolType, + groupProtocols, + }), + { apiVersion: 4 } + ) diff --git a/node_modules/kafkajs/src/protocol/requests/joinGroup/v4/response.js b/node_modules/kafkajs/src/protocol/requests/joinGroup/v4/response.js new file mode 100644 index 0000000..db9ee18 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/joinGroup/v4/response.js @@ -0,0 +1,39 @@ +const { decode } = require('../v3/response') +const { KafkaJSMemberIdRequired } = require('../../../../errors') +const { failure, createErrorFromCode, errorCodes } = require('../../../error') + +/** + * JoinGroup Response (Version: 4) => throttle_time_ms error_code generation_id group_protocol leader_id member_id [members] + * throttle_time_ms => INT32 + * error_code => INT16 + * generation_id => INT32 + * group_protocol => STRING + * leader_id => STRING + * member_id => STRING + * members => member_id member_metadata + * member_id => STRING + * member_metadata => BYTES + */ + +const { code: MEMBER_ID_REQUIRED_ERROR_CODE } = errorCodes.find( + e => e.type === 'MEMBER_ID_REQUIRED' +) + +const parse = async data => { + if (failure(data.errorCode)) { + if (data.errorCode === MEMBER_ID_REQUIRED_ERROR_CODE) { + throw new KafkaJSMemberIdRequired(createErrorFromCode(data.errorCode), { + memberId: data.memberId, + }) + } + + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/joinGroup/v5/request.js b/node_modules/kafkajs/src/protocol/requests/joinGroup/v5/request.js new file mode 100644 index 0000000..a3b33bb --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/joinGroup/v5/request.js @@ -0,0 +1,46 @@ +const Encoder = require('../../../encoder') +const { JoinGroup: apiKey } = require('../../apiKeys') + +/** + * Version 5 adds group_instance_id to identify members across restarts. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances + * + * JoinGroup Request (Version: 5) => group_id session_timeout rebalance_timeout member_id group_instance_id protocol_type [group_protocols] + * group_id => STRING + * session_timeout => INT32 + * rebalance_timeout => INT32 + * member_id => STRING + * group_instance_id => NULLABLE_STRING + * protocol_type => STRING + * group_protocols => protocol_name protocol_metadata + * protocol_name => STRING + * protocol_metadata => BYTES + */ + +module.exports = ({ + groupId, + sessionTimeout, + rebalanceTimeout, + memberId, + groupInstanceId = null, + protocolType, + groupProtocols, +}) => ({ + apiKey, + apiVersion: 5, + apiName: 'JoinGroup', + encode: async () => { + return new Encoder() + .writeString(groupId) + .writeInt32(sessionTimeout) + .writeInt32(rebalanceTimeout) + .writeString(memberId) + .writeString(groupInstanceId) + .writeString(protocolType) + .writeArray(groupProtocols.map(encodeGroupProtocols)) + }, +}) + +const encodeGroupProtocols = ({ name, metadata = Buffer.alloc(0) }) => { + return new Encoder().writeString(name).writeBytes(metadata) +} diff --git a/node_modules/kafkajs/src/protocol/requests/joinGroup/v5/response.js b/node_modules/kafkajs/src/protocol/requests/joinGroup/v5/response.js new file mode 100644 index 0000000..ac0a980 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/joinGroup/v5/response.js @@ -0,0 +1,67 @@ +const Decoder = require('../../../decoder') +const { KafkaJSMemberIdRequired } = require('../../../../errors') +const { + failure, + createErrorFromCode, + errorCodes, + failIfVersionNotSupported, +} = require('../../../error') + +/** + * JoinGroup Response (Version: 5) => throttle_time_ms error_code generation_id group_protocol leader_id member_id [members] + * throttle_time_ms => INT32 + * error_code => INT16 + * generation_id => INT32 + * group_protocol => STRING + * leader_id => STRING + * member_id => STRING + * members => member_id group_instance_id metadata + * member_id => STRING + * group_instance_id => NULLABLE_STRING + * member_metadata => BYTES + */ +const { code: MEMBER_ID_REQUIRED_ERROR_CODE } = errorCodes.find( + e => e.type === 'MEMBER_ID_REQUIRED' +) + +const parse = async data => { + if (failure(data.errorCode)) { + if (data.errorCode === MEMBER_ID_REQUIRED_ERROR_CODE) { + throw new KafkaJSMemberIdRequired(createErrorFromCode(data.errorCode), { + memberId: data.memberId, + }) + } + + throw createErrorFromCode(data.errorCode) + } + + return data +} + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { + throttleTime: 0, + clientSideThrottleTime: throttleTime, + errorCode, + generationId: decoder.readInt32(), + groupProtocol: decoder.readString(), + leaderId: decoder.readString(), + memberId: decoder.readString(), + members: decoder.readArray(decoder => ({ + memberId: decoder.readString(), + groupInstanceId: decoder.readString(), + memberMetadata: decoder.readBytes(), + })), + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/leaveGroup/index.js b/node_modules/kafkajs/src/protocol/requests/leaveGroup/index.js new file mode 100644 index 0000000..9fb9a9c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/leaveGroup/index.js @@ -0,0 +1,39 @@ +const versions = { + 0: ({ groupId, memberId }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { + request: request({ groupId, memberId }), + response, + } + }, + 1: ({ groupId, memberId }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { + request: request({ groupId, memberId }), + response, + } + }, + 2: ({ groupId, memberId }) => { + const request = require('./v2/request') + const response = require('./v2/response') + return { + request: request({ groupId, memberId }), + response, + } + }, + 3: ({ groupId, memberId, groupInstanceId }) => { + const request = require('./v3/request') + const response = require('./v3/response') + return { + request: request({ groupId, members: [{ memberId, groupInstanceId }] }), + response, + } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/leaveGroup/v0/request.js b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v0/request.js new file mode 100644 index 0000000..e17cd3d --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v0/request.js @@ -0,0 +1,17 @@ +const Encoder = require('../../../encoder') +const { LeaveGroup: apiKey } = require('../../apiKeys') + +/** + * LeaveGroup Request (Version: 0) => group_id member_id + * group_id => STRING + * member_id => STRING + */ + +module.exports = ({ groupId, memberId }) => ({ + apiKey, + apiVersion: 0, + apiName: 'LeaveGroup', + encode: async () => { + return new Encoder().writeString(groupId).writeString(memberId) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/leaveGroup/v0/response.js b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v0/response.js new file mode 100644 index 0000000..affbf6f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v0/response.js @@ -0,0 +1,29 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode, failIfVersionNotSupported } = require('../../../error') + +/** + * LeaveGroup Response (Version: 0) => error_code + * error_code => INT16 + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { errorCode } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/leaveGroup/v1/request.js b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v1/request.js new file mode 100644 index 0000000..91c65e1 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v1/request.js @@ -0,0 +1,10 @@ +const requestV0 = require('../v0/request') + +/** + * LeaveGroup Request (Version: 1) => group_id member_id + * group_id => STRING + * member_id => STRING + */ + +module.exports = ({ groupId, memberId }) => + Object.assign(requestV0({ groupId, memberId }), { apiVersion: 1 }) diff --git a/node_modules/kafkajs/src/protocol/requests/leaveGroup/v1/response.js b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v1/response.js new file mode 100644 index 0000000..c8affc2 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v1/response.js @@ -0,0 +1,24 @@ +const Decoder = require('../../../decoder') +const { failIfVersionNotSupported } = require('../../../error') +const { parse: parseV0 } = require('../v0/response') + +/** + * LeaveGroup Response (Version: 1) => throttle_time_ms error_code + * throttle_time_ms => INT32 + * error_code => INT16 + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { throttleTime, errorCode } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/leaveGroup/v2/request.js b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v2/request.js new file mode 100644 index 0000000..5e60b70 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v2/request.js @@ -0,0 +1,10 @@ +const requestV1 = require('../v1/request') + +/** + * LeaveGroup Request (Version: 2) => group_id member_id + * group_id => STRING + * member_id => STRING + */ + +module.exports = ({ groupId, memberId }) => + Object.assign(requestV1({ groupId, memberId }), { apiVersion: 2 }) diff --git a/node_modules/kafkajs/src/protocol/requests/leaveGroup/v2/response.js b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v2/response.js new file mode 100644 index 0000000..4e0179a --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v2/response.js @@ -0,0 +1,24 @@ +const { parse, decode: decodeV1 } = require('../v1/response') + +/** + * In version 2 on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * LeaveGroup Response (Version: 2) => throttle_time_ms error_code + * throttle_time_ms => INT32 + * error_code => INT16 + */ +const decode = async rawData => { + const decoded = await decodeV1(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/leaveGroup/v3/request.js b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v3/request.js new file mode 100644 index 0000000..3874ad3 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v3/request.js @@ -0,0 +1,29 @@ +const Encoder = require('../../../encoder') +const { LeaveGroup: apiKey } = require('../../apiKeys') + +/** + * Version 3 changes leavegroup to operate on a batch of members + * and adds group_instance_id to identify members across restarts. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances + * + * LeaveGroup Request (Version: 3) => group_id [members] + * group_id => STRING + * members => member_id group_instance_id + * member_id => STRING + * group_instance_id => NULLABLE_STRING + */ + +module.exports = ({ groupId, members }) => ({ + apiKey, + apiVersion: 3, + apiName: 'LeaveGroup', + encode: async () => { + return new Encoder() + .writeString(groupId) + .writeArray(members.map(member => encodeMember(member))) + }, +}) + +const encodeMember = ({ memberId, groupInstanceId = null }) => { + return new Encoder().writeString(memberId).writeString(groupInstanceId) +} diff --git a/node_modules/kafkajs/src/protocol/requests/leaveGroup/v3/response.js b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v3/response.js new file mode 100644 index 0000000..449d993 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/leaveGroup/v3/response.js @@ -0,0 +1,46 @@ +const Decoder = require('../../../decoder') +const { failIfVersionNotSupported, failure, createErrorFromCode } = require('../../../error') +const { parse: parseV2 } = require('../v2/response') + +/** + * LeaveGroup Response (Version: 3) => throttle_time_ms error_code [members] + * throttle_time_ms => INT32 + * error_code => INT16 + * members => member_id group_instance_id error_code + * member_id => STRING + * group_instance_id => NULLABLE_STRING + * error_code => INT16 + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + const members = decoder.readArray(decodeMembers) + + failIfVersionNotSupported(errorCode) + + return { throttleTime: 0, clientSideThrottleTime: throttleTime, errorCode, members } +} + +const decodeMembers = decoder => ({ + memberId: decoder.readString(), + groupInstanceId: decoder.readString(), + errorCode: decoder.readInt16(), +}) + +const parse = async data => { + const parsed = parseV2(data) + + const memberWithError = data.members.find(member => failure(member.errorCode)) + if (memberWithError) { + throw createErrorFromCode(memberWithError.errorCode) + } + + return parsed +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/listGroups/index.js b/node_modules/kafkajs/src/protocol/requests/listGroups/index.js new file mode 100644 index 0000000..4ddefbb --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listGroups/index.js @@ -0,0 +1,22 @@ +const versions = { + 0: () => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request(), response } + }, + 1: () => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request(), response } + }, + 2: () => { + const request = require('./v2/request') + const response = require('./v2/response') + return { request: request(), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/listGroups/v0/request.js b/node_modules/kafkajs/src/protocol/requests/listGroups/v0/request.js new file mode 100644 index 0000000..ba1e3b7 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listGroups/v0/request.js @@ -0,0 +1,17 @@ +const Encoder = require('../../../encoder') +const { ListGroups: apiKey } = require('../../apiKeys') + +/** + * ListGroups Request (Version: 0) + */ + +/** + */ +module.exports = () => ({ + apiKey, + apiVersion: 0, + apiName: 'ListGroups', + encode: async () => { + return new Encoder() + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/listGroups/v0/response.js b/node_modules/kafkajs/src/protocol/requests/listGroups/v0/response.js new file mode 100644 index 0000000..4cc2650 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listGroups/v0/response.js @@ -0,0 +1,40 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * ListGroups Response (Version: 0) => error_code [groups] + * error_code => INT16 + * groups => group_id protocol_type + * group_id => STRING + * protocol_type => STRING + */ + +const decodeGroup = decoder => ({ + groupId: decoder.readString(), + protocolType: decoder.readString(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const errorCode = decoder.readInt16() + const groups = decoder.readArray(decodeGroup) + + return { + errorCode, + groups, + } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decodeGroup, + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/listGroups/v1/request.js b/node_modules/kafkajs/src/protocol/requests/listGroups/v1/request.js new file mode 100644 index 0000000..84900f4 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listGroups/v1/request.js @@ -0,0 +1,7 @@ +const requestV0 = require('../v0/request') + +/** + * ListGroups Request (Version: 1) + */ + +module.exports = () => Object.assign(requestV0(), { apiVersion: 1 }) diff --git a/node_modules/kafkajs/src/protocol/requests/listGroups/v1/response.js b/node_modules/kafkajs/src/protocol/requests/listGroups/v1/response.js new file mode 100644 index 0000000..950850a --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listGroups/v1/response.js @@ -0,0 +1,30 @@ +const responseV0 = require('../v0/response') + +const Decoder = require('../../../decoder') + +/** + * ListGroups Response (Version: 1) => error_code [groups] + * throttle_time_ms => INT32 + * error_code => INT16 + * groups => group_id protocol_type + * group_id => STRING + * protocol_type => STRING + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + const groups = decoder.readArray(responseV0.decodeGroup) + + return { + throttleTime, + errorCode, + groups, + } +} + +module.exports = { + decode, + parse: responseV0.parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/listGroups/v2/request.js b/node_modules/kafkajs/src/protocol/requests/listGroups/v2/request.js new file mode 100644 index 0000000..9d4fda6 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listGroups/v2/request.js @@ -0,0 +1,7 @@ +const requestV1 = require('../v1/request') + +/** + * ListGroups Request (Version: 2) + */ + +module.exports = () => Object.assign(requestV1(), { apiVersion: 2 }) diff --git a/node_modules/kafkajs/src/protocol/requests/listGroups/v2/response.js b/node_modules/kafkajs/src/protocol/requests/listGroups/v2/response.js new file mode 100644 index 0000000..aa694b8 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listGroups/v2/response.js @@ -0,0 +1,27 @@ +const { parse, decode: decodeV1 } = require('../v1/response') + +/** + * In version 2 on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * ListGroups Response (Version: 2) => error_code [groups] + * throttle_time_ms => INT32 + * error_code => INT16 + * groups => group_id protocol_type + * group_id => STRING + * protocol_type => STRING + */ +const decode = async rawData => { + const decoded = await decodeV1(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/listOffsets/index.js b/node_modules/kafkajs/src/protocol/requests/listOffsets/index.js new file mode 100644 index 0000000..ac93d93 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listOffsets/index.js @@ -0,0 +1,32 @@ +const ISOLATION_LEVEL = require('../../isolationLevel') + +// For normal consumers, use -1 +const REPLICA_ID = -1 + +const versions = { + 0: ({ replicaId = REPLICA_ID, topics }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ replicaId, topics }), response } + }, + 1: ({ replicaId = REPLICA_ID, topics }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ replicaId, topics }), response } + }, + 2: ({ replicaId = REPLICA_ID, isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, topics }) => { + const request = require('./v2/request') + const response = require('./v2/response') + return { request: request({ replicaId, isolationLevel, topics }), response } + }, + 3: ({ replicaId = REPLICA_ID, isolationLevel = ISOLATION_LEVEL.READ_COMMITTED, topics }) => { + const request = require('./v3/request') + const response = require('./v3/response') + return { request: request({ replicaId, isolationLevel, topics }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/listOffsets/v0/request.js b/node_modules/kafkajs/src/protocol/requests/listOffsets/v0/request.js new file mode 100644 index 0000000..338b411 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listOffsets/v0/request.js @@ -0,0 +1,46 @@ +const Encoder = require('../../../encoder') +const { ListOffsets: apiKey } = require('../../apiKeys') + +/** + * ListOffsets Request (Version: 0) => replica_id [topics] + * replica_id => INT32 + * topics => topic [partitions] + * topic => STRING + * partitions => partition timestamp max_num_offsets + * partition => INT32 + * timestamp => INT64 + * max_num_offsets => INT32 + */ + +/** + * @param {number} replicaId + * @param {object} topics use timestamp=-1 for latest offsets and timestamp=-2 for earliest. + * Default timestamp=-1. Example: + * { + * topics: [ + * { + * topic: 'topic-name', + * partitions: [{ partition: 0, timestamp: -1 }] + * } + * ] + * } + */ +module.exports = ({ replicaId, topics }) => ({ + apiKey, + apiVersion: 0, + apiName: 'ListOffsets', + encode: async () => { + return new Encoder().writeInt32(replicaId).writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition, timestamp = -1, maxNumOffsets = 1 }) => { + return new Encoder() + .writeInt32(partition) + .writeInt64(timestamp) + .writeInt32(maxNumOffsets) +} diff --git a/node_modules/kafkajs/src/protocol/requests/listOffsets/v0/response.js b/node_modules/kafkajs/src/protocol/requests/listOffsets/v0/response.js new file mode 100644 index 0000000..3fb4c66 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listOffsets/v0/response.js @@ -0,0 +1,49 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * Offsets Response (Version: 0) => [responses] + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code [offsets] + * partition => INT32 + * error_code => INT16 + * offsets => INT64 + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + responses: decoder.readArray(decodeResponses), + } +} + +const decodeResponses = decoder => ({ + topic: decoder.readString(), + partitions: decoder.readArray(decodePartitions), +}) + +const decodePartitions = decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + offsets: decoder.readArray(decodeOffsets), +}) + +const decodeOffsets = decoder => decoder.readInt64().toString() + +const parse = async data => { + const partitionsWithError = data.responses.flatMap(response => + response.partitions.filter(partition => failure(partition.errorCode)) + ) + const partitionWithError = partitionsWithError[0] + if (partitionWithError) { + throw createErrorFromCode(partitionWithError.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/listOffsets/v1/request.js b/node_modules/kafkajs/src/protocol/requests/listOffsets/v1/request.js new file mode 100644 index 0000000..08e926d --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listOffsets/v1/request.js @@ -0,0 +1,28 @@ +const Encoder = require('../../../encoder') +const { ListOffsets: apiKey } = require('../../apiKeys') + +/** + * ListOffsets Request (Version: 1) => replica_id [topics] + * replica_id => INT32 + * topics => topic [partitions] + * topic => STRING + * partitions => partition timestamp + * partition => INT32 + * timestamp => INT64 + */ +module.exports = ({ replicaId, topics }) => ({ + apiKey, + apiVersion: 1, + apiName: 'ListOffsets', + encode: async () => { + return new Encoder().writeInt32(replicaId).writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition, timestamp = -1 }) => { + return new Encoder().writeInt32(partition).writeInt64(timestamp) +} diff --git a/node_modules/kafkajs/src/protocol/requests/listOffsets/v1/response.js b/node_modules/kafkajs/src/protocol/requests/listOffsets/v1/response.js new file mode 100644 index 0000000..e8b731e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listOffsets/v1/response.js @@ -0,0 +1,49 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * ListOffsets Response (Version: 1) => [responses] + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code timestamp offset + * partition => INT32 + * error_code => INT16 + * timestamp => INT64 + * offset => INT64 + */ +const decode = async rawData => { + const decoder = new Decoder(rawData) + + return { + responses: decoder.readArray(decodeResponses), + } +} + +const decodeResponses = decoder => ({ + topic: decoder.readString(), + partitions: decoder.readArray(decodePartitions), +}) + +const decodePartitions = decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + timestamp: decoder.readInt64().toString(), + offset: decoder.readInt64().toString(), +}) + +const parse = async data => { + const partitionsWithError = data.responses.flatMap(response => + response.partitions.filter(partition => failure(partition.errorCode)) + ) + const partitionWithError = partitionsWithError[0] + if (partitionWithError) { + throw createErrorFromCode(partitionWithError.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/listOffsets/v2/request.js b/node_modules/kafkajs/src/protocol/requests/listOffsets/v2/request.js new file mode 100644 index 0000000..3a8caa0 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listOffsets/v2/request.js @@ -0,0 +1,32 @@ +const Encoder = require('../../../encoder') +const { ListOffsets: apiKey } = require('../../apiKeys') + +/** + * ListOffsets Request (Version: 2) => replica_id isolation_level [topics] + * replica_id => INT32 + * isolation_level => INT8 + * topics => topic [partitions] + * topic => STRING + * partitions => partition timestamp + * partition => INT32 + * timestamp => INT64 + */ +module.exports = ({ replicaId, isolationLevel, topics }) => ({ + apiKey, + apiVersion: 2, + apiName: 'ListOffsets', + encode: async () => { + return new Encoder() + .writeInt32(replicaId) + .writeInt8(isolationLevel) + .writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition, timestamp = -1 }) => { + return new Encoder().writeInt32(partition).writeInt64(timestamp) +} diff --git a/node_modules/kafkajs/src/protocol/requests/listOffsets/v2/response.js b/node_modules/kafkajs/src/protocol/requests/listOffsets/v2/response.js new file mode 100644 index 0000000..a38b85c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listOffsets/v2/response.js @@ -0,0 +1,51 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * ListOffsets Response (Version: 2) => throttle_time_ms [responses] + * throttle_time_ms => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code timestamp offset + * partition => INT32 + * error_code => INT16 + * timestamp => INT64 + * offset => INT64 + */ +const decode = async rawData => { + const decoder = new Decoder(rawData) + + return { + throttleTime: decoder.readInt32(), + responses: decoder.readArray(decodeResponses), + } +} + +const decodeResponses = decoder => ({ + topic: decoder.readString(), + partitions: decoder.readArray(decodePartitions), +}) + +const decodePartitions = decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + timestamp: decoder.readInt64().toString(), + offset: decoder.readInt64().toString(), +}) + +const parse = async data => { + const partitionsWithError = data.responses.flatMap(response => + response.partitions.filter(partition => failure(partition.errorCode)) + ) + const partitionWithError = partitionsWithError[0] + if (partitionWithError) { + throw createErrorFromCode(partitionWithError.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/listOffsets/v3/request.js b/node_modules/kafkajs/src/protocol/requests/listOffsets/v3/request.js new file mode 100644 index 0000000..4bb990d --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listOffsets/v3/request.js @@ -0,0 +1,14 @@ +const requestV2 = require('../v2/request') + +/** + * ListOffsets Request (Version: 3) => replica_id isolation_level [topics] + * replica_id => INT32 + * isolation_level => INT8 + * topics => topic [partitions] + * topic => STRING + * partitions => partition timestamp + * partition => INT32 + * timestamp => INT64 + */ +module.exports = ({ replicaId, isolationLevel, topics }) => + Object.assign(requestV2({ replicaId, isolationLevel, topics }), { apiVersion: 3 }) diff --git a/node_modules/kafkajs/src/protocol/requests/listOffsets/v3/response.js b/node_modules/kafkajs/src/protocol/requests/listOffsets/v3/response.js new file mode 100644 index 0000000..5b2d900 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listOffsets/v3/response.js @@ -0,0 +1,30 @@ +const { parse, decode: decodeV2 } = require('../v2/response') + +/** + * In version 3 on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * ListOffsets Response (Version: 3) => throttle_time_ms [responses] + * throttle_time_ms => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code timestamp offset + * partition => INT32 + * error_code => INT16 + * timestamp => INT64 + * offset => INT64 + */ +const decode = async rawData => { + const decoded = await decodeV2(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/listPartitionReassignments/index.js b/node_modules/kafkajs/src/protocol/requests/listPartitionReassignments/index.js new file mode 100644 index 0000000..40847d6 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listPartitionReassignments/index.js @@ -0,0 +1,12 @@ +const versions = { + 0: ({ topics, timeout }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ topics, timeout }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/listPartitionReassignments/v0/request.js b/node_modules/kafkajs/src/protocol/requests/listPartitionReassignments/v0/request.js new file mode 100644 index 0000000..f596539 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listPartitionReassignments/v0/request.js @@ -0,0 +1,34 @@ +const Encoder = require('../../../encoder') +const { ListPartitionReassignments: apiKey } = require('../../apiKeys') + +/** + * ListPartitionReassignments Request (Version: 0) => timeout_ms [topics] TAG_BUFFER + * timeout_ms => INT32 + * topics => name [partition_indexes] TAG_BUFFER + * name => COMPACT_STRING + * partition_indexes => INT32 + */ + +module.exports = ({ topics = null, timeout = 5000 }) => ({ + apiKey, + apiVersion: 0, + apiName: 'ListPartitionReassignments', + encode: async () => { + return new Encoder() + .writeUVarIntBytes() + .writeInt32(timeout) + .writeUVarIntArray(topics === null ? topics : topics.map(encodeTopics)) + .writeUVarIntBytes() + }, +}) + +const encodeTopics = ({ topic, partitions }) => { + return new Encoder() + .writeUVarIntString(topic) + .writeUVarIntArray(partitions.map(encodePartitions)) + .writeUVarIntBytes() +} + +const encodePartitions = partition => { + return new Encoder().writeInt32(partition) +} diff --git a/node_modules/kafkajs/src/protocol/requests/listPartitionReassignments/v0/response.js b/node_modules/kafkajs/src/protocol/requests/listPartitionReassignments/v0/response.js new file mode 100644 index 0000000..a5576c9 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/listPartitionReassignments/v0/response.js @@ -0,0 +1,72 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * ListPartitionReassignments Response (Version: 0) => throttle_time_ms error_code error_message [topics] TAG_BUFFER + * throttle_time_ms => INT32 + * error_code => INT16 + * error_message => COMPACT_NULLABLE_STRING + * topics => name [partitions] TAG_BUFFER + * name => COMPACT_STRING + * partitions => partition_index [replicas] [adding_replicas] [removing_replicas] TAG_BUFFER + * partition_index => INT32 + * replicas => INT32 + * adding_replicas => INT32 + * removing_replicas => INT32 + */ + +const decodeReplicas = decoder => { + return decoder.readInt32() +} + +const decodePartitions = decoder => { + const partition = { + partition: decoder.readInt32(), + replicas: decoder.readUVarIntArray(decodeReplicas), + addingReplicas: decoder.readUVarIntArray(decodeReplicas), + removingReplicas: decoder.readUVarIntArray(decodeReplicas), + } + + // Read tagged fields + decoder.readTaggedFields() + return partition +} + +const decodeTopics = decoder => { + const topic = { + name: decoder.readUVarIntString(), + partitions: decoder.readUVarIntArray(decodePartitions), + } + + // Read tagged fields + decoder.readTaggedFields() + return topic +} +const decode = async rawData => { + const decoder = new Decoder(rawData) + + // Read tagged fields + decoder.readTaggedFields() + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + // Read error message + decoder.readUVarIntString() + return { + throttleTime, + errorCode, + topics: decoder.readUVarIntArray(decodeTopics), + } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/index.js b/node_modules/kafkajs/src/protocol/requests/metadata/index.js new file mode 100644 index 0000000..8ffdc74 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/index.js @@ -0,0 +1,42 @@ +const versions = { + 0: ({ topics }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ topics }), response } + }, + 1: ({ topics }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ topics }), response } + }, + 2: ({ topics }) => { + const request = require('./v2/request') + const response = require('./v2/response') + return { request: request({ topics }), response } + }, + 3: ({ topics }) => { + const request = require('./v3/request') + const response = require('./v3/response') + return { request: request({ topics }), response } + }, + 4: ({ topics, allowAutoTopicCreation }) => { + const request = require('./v4/request') + const response = require('./v4/response') + return { request: request({ topics, allowAutoTopicCreation }), response } + }, + 5: ({ topics, allowAutoTopicCreation }) => { + const request = require('./v5/request') + const response = require('./v5/response') + return { request: request({ topics, allowAutoTopicCreation }), response } + }, + 6: ({ topics, allowAutoTopicCreation }) => { + const request = require('./v6/request') + const response = require('./v6/response') + return { request: request({ topics, allowAutoTopicCreation }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v0/request.js b/node_modules/kafkajs/src/protocol/requests/metadata/v0/request.js new file mode 100644 index 0000000..6db2c48 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v0/request.js @@ -0,0 +1,16 @@ +const Encoder = require('../../../encoder') +const { Metadata: apiKey } = require('../../apiKeys') + +/** + * Metadata Request (Version: 0) => [topics] + * topics => STRING + */ + +module.exports = ({ topics }) => ({ + apiKey, + apiVersion: 0, + apiName: 'Metadata', + encode: async () => { + return new Encoder().writeArray(topics) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v0/response.js b/node_modules/kafkajs/src/protocol/requests/metadata/v0/response.js new file mode 100644 index 0000000..3b2fb76 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v0/response.js @@ -0,0 +1,73 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * Metadata Response (Version: 0) => [brokers] [topic_metadata] + * brokers => node_id host port + * node_id => INT32 + * host => STRING + * port => INT32 + * topic_metadata => topic_error_code topic [partition_metadata] + * topic_error_code => INT16 + * topic => STRING + * partition_metadata => partition_error_code partition_id leader [replicas] [isr] + * partition_error_code => INT16 + * partition_id => INT32 + * leader => INT32 + * replicas => INT32 + * isr => INT32 + */ + +const broker = decoder => ({ + nodeId: decoder.readInt32(), + host: decoder.readString(), + port: decoder.readInt32(), +}) + +const topicMetadata = decoder => ({ + topicErrorCode: decoder.readInt16(), + topic: decoder.readString(), + partitionMetadata: decoder.readArray(partitionMetadata), +}) + +const partitionMetadata = decoder => ({ + partitionErrorCode: decoder.readInt16(), + partitionId: decoder.readInt32(), + // leader: The node id for the kafka broker currently acting as leader + // for this partition + leader: decoder.readInt32(), + replicas: decoder.readArray(d => d.readInt32()), + isr: decoder.readArray(d => d.readInt32()), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + brokers: decoder.readArray(broker), + topicMetadata: decoder.readArray(topicMetadata), + } +} + +const parse = async data => { + const topicsWithErrors = data.topicMetadata.filter(topic => failure(topic.topicErrorCode)) + if (topicsWithErrors.length > 0) { + const { topicErrorCode } = topicsWithErrors[0] + throw createErrorFromCode(topicErrorCode) + } + + const errors = data.topicMetadata.flatMap(topic => { + return topic.partitionMetadata.filter(partition => failure(partition.partitionErrorCode)) + }) + + if (errors.length > 0) { + const { partitionErrorCode } = errors[0] + throw createErrorFromCode(partitionErrorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v1/request.js b/node_modules/kafkajs/src/protocol/requests/metadata/v1/request.js new file mode 100644 index 0000000..9dc4911 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v1/request.js @@ -0,0 +1,16 @@ +const Encoder = require('../../../encoder') +const { Metadata: apiKey } = require('../../apiKeys') + +/** + * Metadata Request (Version: 1) => [topics] + * topics => STRING + */ + +module.exports = ({ topics }) => ({ + apiKey, + apiVersion: 1, + apiName: 'Metadata', + encode: async () => { + return new Encoder().writeNullableArray(topics) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v1/response.js b/node_modules/kafkajs/src/protocol/requests/metadata/v1/response.js new file mode 100644 index 0000000..ebf8b36 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v1/response.js @@ -0,0 +1,58 @@ +const Decoder = require('../../../decoder') +const { parse: parseV0 } = require('../v0/response') + +/** + * Metadata Response (Version: 1) => [brokers] controller_id [topic_metadata] + * brokers => node_id host port rack + * node_id => INT32 + * host => STRING + * port => INT32 + * rack => NULLABLE_STRING + * controller_id => INT32 + * topic_metadata => topic_error_code topic is_internal [partition_metadata] + * topic_error_code => INT16 + * topic => STRING + * is_internal => BOOLEAN + * partition_metadata => partition_error_code partition_id leader [replicas] [isr] + * partition_error_code => INT16 + * partition_id => INT32 + * leader => INT32 + * replicas => INT32 + * isr => INT32 + */ + +const broker = decoder => ({ + nodeId: decoder.readInt32(), + host: decoder.readString(), + port: decoder.readInt32(), + rack: decoder.readString(), +}) + +const topicMetadata = decoder => ({ + topicErrorCode: decoder.readInt16(), + topic: decoder.readString(), + isInternal: decoder.readBoolean(), + partitionMetadata: decoder.readArray(partitionMetadata), +}) + +const partitionMetadata = decoder => ({ + partitionErrorCode: decoder.readInt16(), + partitionId: decoder.readInt32(), + leader: decoder.readInt32(), + replicas: decoder.readArray(d => d.readInt32()), + isr: decoder.readArray(d => d.readInt32()), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + brokers: decoder.readArray(broker), + controllerId: decoder.readInt32(), + topicMetadata: decoder.readArray(topicMetadata), + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v2/request.js b/node_modules/kafkajs/src/protocol/requests/metadata/v2/request.js new file mode 100644 index 0000000..9d7f55e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v2/request.js @@ -0,0 +1,8 @@ +const requestV1 = require('../v1/request') + +/** + * Metadata Request (Version: 2) => [topics] + * topics => STRING + */ + +module.exports = ({ topics }) => Object.assign(requestV1({ topics }), { apiVersion: 2 }) diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v2/response.js b/node_modules/kafkajs/src/protocol/requests/metadata/v2/response.js new file mode 100644 index 0000000..269749e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v2/response.js @@ -0,0 +1,60 @@ +const Decoder = require('../../../decoder') +const { parse: parseV0 } = require('../v0/response') + +/** + * Metadata Response (Version: 2) => [brokers] cluster_id controller_id [topic_metadata] + * brokers => node_id host port rack + * node_id => INT32 + * host => STRING + * port => INT32 + * rack => NULLABLE_STRING + * cluster_id => NULLABLE_STRING + * controller_id => INT32 + * topic_metadata => topic_error_code topic is_internal [partition_metadata] + * topic_error_code => INT16 + * topic => STRING + * is_internal => BOOLEAN + * partition_metadata => partition_error_code partition_id leader [replicas] [isr] + * partition_error_code => INT16 + * partition_id => INT32 + * leader => INT32 + * replicas => INT32 + * isr => INT32 + */ + +const broker = decoder => ({ + nodeId: decoder.readInt32(), + host: decoder.readString(), + port: decoder.readInt32(), + rack: decoder.readString(), +}) + +const topicMetadata = decoder => ({ + topicErrorCode: decoder.readInt16(), + topic: decoder.readString(), + isInternal: decoder.readBoolean(), + partitionMetadata: decoder.readArray(partitionMetadata), +}) + +const partitionMetadata = decoder => ({ + partitionErrorCode: decoder.readInt16(), + partitionId: decoder.readInt32(), + leader: decoder.readInt32(), + replicas: decoder.readArray(d => d.readInt32()), + isr: decoder.readArray(d => d.readInt32()), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + brokers: decoder.readArray(broker), + clusterId: decoder.readString(), + controllerId: decoder.readInt32(), + topicMetadata: decoder.readArray(topicMetadata), + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v3/request.js b/node_modules/kafkajs/src/protocol/requests/metadata/v3/request.js new file mode 100644 index 0000000..b5936bf --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v3/request.js @@ -0,0 +1,8 @@ +const requestV1 = require('../v1/request') + +/** + * Metadata Request (Version: 3) => [topics] + * topics => STRING + */ + +module.exports = ({ topics }) => Object.assign(requestV1({ topics }), { apiVersion: 3 }) diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v3/response.js b/node_modules/kafkajs/src/protocol/requests/metadata/v3/response.js new file mode 100644 index 0000000..f1c16d3 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v3/response.js @@ -0,0 +1,62 @@ +const Decoder = require('../../../decoder') +const { parse: parseV0 } = require('../v0/response') + +/** + * Metadata Response (Version: 3) => throttle_time_ms [brokers] cluster_id controller_id [topic_metadata] + * throttle_time_ms => INT32 + * brokers => node_id host port rack + * node_id => INT32 + * host => STRING + * port => INT32 + * rack => NULLABLE_STRING + * cluster_id => NULLABLE_STRING + * controller_id => INT32 + * topic_metadata => error_code topic is_internal [partition_metadata] + * error_code => INT16 + * topic => STRING + * is_internal => BOOLEAN + * partition_metadata => error_code partition leader [replicas] [isr] + * error_code => INT16 + * partition => INT32 + * leader => INT32 + * replicas => INT32 + * isr => INT32 + */ + +const broker = decoder => ({ + nodeId: decoder.readInt32(), + host: decoder.readString(), + port: decoder.readInt32(), + rack: decoder.readString(), +}) + +const topicMetadata = decoder => ({ + topicErrorCode: decoder.readInt16(), + topic: decoder.readString(), + isInternal: decoder.readBoolean(), + partitionMetadata: decoder.readArray(partitionMetadata), +}) + +const partitionMetadata = decoder => ({ + partitionErrorCode: decoder.readInt16(), + partitionId: decoder.readInt32(), + leader: decoder.readInt32(), + replicas: decoder.readArray(d => d.readInt32()), + isr: decoder.readArray(d => d.readInt32()), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + throttleTime: decoder.readInt32(), + brokers: decoder.readArray(broker), + clusterId: decoder.readString(), + controllerId: decoder.readInt32(), + topicMetadata: decoder.readArray(topicMetadata), + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v4/request.js b/node_modules/kafkajs/src/protocol/requests/metadata/v4/request.js new file mode 100644 index 0000000..97c3829 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v4/request.js @@ -0,0 +1,17 @@ +const Encoder = require('../../../encoder') +const { Metadata: apiKey } = require('../../apiKeys') + +/** + * Metadata Request (Version: 4) => [topics] allow_auto_topic_creation + * topics => STRING + * allow_auto_topic_creation => BOOLEAN + */ + +module.exports = ({ topics, allowAutoTopicCreation = true }) => ({ + apiKey, + apiVersion: 4, + apiName: 'Metadata', + encode: async () => { + return new Encoder().writeNullableArray(topics).writeBoolean(allowAutoTopicCreation) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v4/response.js b/node_modules/kafkajs/src/protocol/requests/metadata/v4/response.js new file mode 100644 index 0000000..878f06f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v4/response.js @@ -0,0 +1,28 @@ +const { parse: parseV3, decode: decodeV3 } = require('../v3/response') + +/** + * Metadata Response (Version: 4) => throttle_time_ms [brokers] cluster_id controller_id [topic_metadata] + * throttle_time_ms => INT32 + * brokers => node_id host port rack + * node_id => INT32 + * host => STRING + * port => INT32 + * rack => NULLABLE_STRING + * cluster_id => NULLABLE_STRING + * controller_id => INT32 + * topic_metadata => error_code topic is_internal [partition_metadata] + * error_code => INT16 + * topic => STRING + * is_internal => BOOLEAN + * partition_metadata => error_code partition leader [replicas] [isr] + * error_code => INT16 + * partition => INT32 + * leader => INT32 + * replicas => INT32 + * isr => INT32 + */ + +module.exports = { + parse: parseV3, + decode: decodeV3, +} diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v5/request.js b/node_modules/kafkajs/src/protocol/requests/metadata/v5/request.js new file mode 100644 index 0000000..856dd56 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v5/request.js @@ -0,0 +1,10 @@ +const requestV4 = require('../v4/request') + +/** + * Metadata Request (Version: 5) => [topics] allow_auto_topic_creation + * topics => STRING + * allow_auto_topic_creation => BOOLEAN + */ + +module.exports = ({ topics, allowAutoTopicCreation = true }) => + Object.assign(requestV4({ topics, allowAutoTopicCreation }), { apiVersion: 5 }) diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v5/response.js b/node_modules/kafkajs/src/protocol/requests/metadata/v5/response.js new file mode 100644 index 0000000..1d94b8f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v5/response.js @@ -0,0 +1,64 @@ +const Decoder = require('../../../decoder') +const { parse: parseV0 } = require('../v0/response') + +/** + * Metadata Response (Version: 5) => throttle_time_ms [brokers] cluster_id controller_id [topic_metadata] + * throttle_time_ms => INT32 + * brokers => node_id host port rack + * node_id => INT32 + * host => STRING + * port => INT32 + * rack => NULLABLE_STRING + * cluster_id => NULLABLE_STRING + * controller_id => INT32 + * topic_metadata => error_code topic is_internal [partition_metadata] + * error_code => INT16 + * topic => STRING + * is_internal => BOOLEAN + * partition_metadata => error_code partition leader [replicas] [isr] [offline_replicas] + * error_code => INT16 + * partition => INT32 + * leader => INT32 + * replicas => INT32 + * isr => INT32 + * offline_replicas => INT32 + */ + +const broker = decoder => ({ + nodeId: decoder.readInt32(), + host: decoder.readString(), + port: decoder.readInt32(), + rack: decoder.readString(), +}) + +const topicMetadata = decoder => ({ + topicErrorCode: decoder.readInt16(), + topic: decoder.readString(), + isInternal: decoder.readBoolean(), + partitionMetadata: decoder.readArray(partitionMetadata), +}) + +const partitionMetadata = decoder => ({ + partitionErrorCode: decoder.readInt16(), + partitionId: decoder.readInt32(), + leader: decoder.readInt32(), + replicas: decoder.readArray(d => d.readInt32()), + isr: decoder.readArray(d => d.readInt32()), + offlineReplicas: decoder.readArray(d => d.readInt32()), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + throttleTime: decoder.readInt32(), + brokers: decoder.readArray(broker), + clusterId: decoder.readString(), + controllerId: decoder.readInt32(), + topicMetadata: decoder.readArray(topicMetadata), + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v6/request.js b/node_modules/kafkajs/src/protocol/requests/metadata/v6/request.js new file mode 100644 index 0000000..c2cd591 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v6/request.js @@ -0,0 +1,10 @@ +const requestV5 = require('../v5/request') + +/** + * Metadata Request (Version: 6) => [topics] allow_auto_topic_creation + * topics => STRING + * allow_auto_topic_creation => BOOLEAN + */ + +module.exports = ({ topics, allowAutoTopicCreation = true }) => + Object.assign(requestV5({ topics, allowAutoTopicCreation }), { apiVersion: 6 }) diff --git a/node_modules/kafkajs/src/protocol/requests/metadata/v6/response.js b/node_modules/kafkajs/src/protocol/requests/metadata/v6/response.js new file mode 100644 index 0000000..317dcc4 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/metadata/v6/response.js @@ -0,0 +1,41 @@ +const { parse, decode: decodeV1 } = require('../v5/response') + +/** + * In version 6 on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * Metadata Response (Version: 6) => throttle_time_ms [brokers] cluster_id controller_id [topic_metadata] + * throttle_time_ms => INT32 + * brokers => node_id host port rack + * node_id => INT32 + * host => STRING + * port => INT32 + * rack => NULLABLE_STRING + * cluster_id => NULLABLE_STRING + * controller_id => INT32 + * topic_metadata => error_code topic is_internal [partition_metadata] + * error_code => INT16 + * topic => STRING + * is_internal => BOOLEAN + * partition_metadata => error_code partition leader [replicas] [isr] [offline_replicas] + * error_code => INT16 + * partition => INT32 + * leader => INT32 + * replicas => INT32 + * isr => INT32 + * offline_replicas => INT32 + */ +const decode = async rawData => { + const decoded = await decodeV1(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetCommit/index.js b/node_modules/kafkajs/src/protocol/requests/offsetCommit/index.js new file mode 100644 index 0000000..ee9a22e --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetCommit/index.js @@ -0,0 +1,75 @@ +// This value signals to the broker that its default configuration should be used. +const RETENTION_TIME = -1 + +const versions = { + 0: ({ groupId, topics }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ groupId, topics }), response } + }, + 1: ({ groupId, groupGenerationId, memberId, topics }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ groupId, groupGenerationId, memberId, topics }), response } + }, + 2: ({ groupId, groupGenerationId, memberId, retentionTime = RETENTION_TIME, topics }) => { + const request = require('./v2/request') + const response = require('./v2/response') + return { + request: request({ + groupId, + groupGenerationId, + memberId, + retentionTime, + topics, + }), + response, + } + }, + 3: ({ groupId, groupGenerationId, memberId, retentionTime = RETENTION_TIME, topics }) => { + const request = require('./v3/request') + const response = require('./v3/response') + return { + request: request({ + groupId, + groupGenerationId, + memberId, + retentionTime, + topics, + }), + response, + } + }, + 4: ({ groupId, groupGenerationId, memberId, retentionTime = RETENTION_TIME, topics }) => { + const request = require('./v4/request') + const response = require('./v4/response') + return { + request: request({ + groupId, + groupGenerationId, + memberId, + retentionTime, + topics, + }), + response, + } + }, + 5: ({ groupId, groupGenerationId, memberId, topics }) => { + const request = require('./v5/request') + const response = require('./v5/response') + return { + request: request({ + groupId, + groupGenerationId, + memberId, + topics, + }), + response, + } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetCommit/v0/request.js b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v0/request.js new file mode 100644 index 0000000..956f342 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v0/request.js @@ -0,0 +1,33 @@ +const Encoder = require('../../../encoder') +const { OffsetCommit: apiKey } = require('../../apiKeys') + +/** + * OffsetCommit Request (Version: 0) => group_id [topics] + * group_id => STRING + * topics => topic [partitions] + * topic => STRING + * partitions => partition offset metadata + * partition => INT32 + * offset => INT64 + * metadata => NULLABLE_STRING + */ + +module.exports = ({ groupId, topics }) => ({ + apiKey, + apiVersion: 0, + apiName: 'OffsetCommit', + encode: async () => { + return new Encoder().writeString(groupId).writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition, offset, metadata = null }) => { + return new Encoder() + .writeInt32(partition) + .writeInt64(offset) + .writeString(metadata) +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetCommit/v0/response.js b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v0/response.js new file mode 100644 index 0000000..0a50e86 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v0/response.js @@ -0,0 +1,45 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * OffsetCommit Response (Version: 0) => [responses] + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code + * partition => INT32 + * error_code => INT16 + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + responses: decoder.readArray(decodeResponses), + } +} + +const decodeResponses = decoder => ({ + topic: decoder.readString(), + partitions: decoder.readArray(decodePartitions), +}) + +const decodePartitions = decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), +}) + +const parse = async data => { + const partitionsWithError = data.responses.flatMap(response => + response.partitions.filter(partition => failure(partition.errorCode)) + ) + const partitionWithError = partitionsWithError[0] + if (partitionWithError) { + throw createErrorFromCode(partitionWithError.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetCommit/v1/request.js b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v1/request.js new file mode 100644 index 0000000..7d8cb91 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v1/request.js @@ -0,0 +1,41 @@ +const Encoder = require('../../../encoder') +const { OffsetCommit: apiKey } = require('../../apiKeys') + +/** + * OffsetCommit Request (Version: 1) => group_id group_generation_id member_id [topics] + * group_id => STRING + * group_generation_id => INT32 + * member_id => STRING + * topics => topic [partitions] + * topic => STRING + * partitions => partition offset timestamp metadata + * partition => INT32 + * offset => INT64 + * timestamp => INT64 + * metadata => NULLABLE_STRING + */ + +module.exports = ({ groupId, groupGenerationId, memberId, topics }) => ({ + apiKey, + apiVersion: 1, + apiName: 'OffsetCommit', + encode: async () => { + return new Encoder() + .writeString(groupId) + .writeInt32(groupGenerationId) + .writeString(memberId) + .writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition, offset, timestamp = Date.now(), metadata = null }) => { + return new Encoder() + .writeInt32(partition) + .writeInt64(offset) + .writeInt64(timestamp) + .writeString(metadata) +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetCommit/v1/response.js b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v1/response.js new file mode 100644 index 0000000..364b7e1 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v1/response.js @@ -0,0 +1,15 @@ +const { parse, decode } = require('../v0/response') + +/** + * OffsetCommit Response (Version: 1) => [responses] + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code + * partition => INT32 + * error_code => INT16 + */ + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetCommit/v2/request.js b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v2/request.js new file mode 100644 index 0000000..39c905f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v2/request.js @@ -0,0 +1,41 @@ +const Encoder = require('../../../encoder') +const { OffsetCommit: apiKey } = require('../../apiKeys') + +/** + * OffsetCommit Request (Version: 2) => group_id group_generation_id member_id retention_time [topics] + * group_id => STRING + * group_generation_id => INT32 + * member_id => STRING + * retention_time => INT64 + * topics => topic [partitions] + * topic => STRING + * partitions => partition offset metadata + * partition => INT32 + * offset => INT64 + * metadata => NULLABLE_STRING + */ + +module.exports = ({ groupId, groupGenerationId, memberId, retentionTime, topics }) => ({ + apiKey, + apiVersion: 2, + apiName: 'OffsetCommit', + encode: async () => { + return new Encoder() + .writeString(groupId) + .writeInt32(groupGenerationId) + .writeString(memberId) + .writeInt64(retentionTime) + .writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition, offset, metadata = null }) => { + return new Encoder() + .writeInt32(partition) + .writeInt64(offset) + .writeString(metadata) +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetCommit/v2/response.js b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v2/response.js new file mode 100644 index 0000000..364b7e1 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v2/response.js @@ -0,0 +1,15 @@ +const { parse, decode } = require('../v0/response') + +/** + * OffsetCommit Response (Version: 1) => [responses] + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code + * partition => INT32 + * error_code => INT16 + */ + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetCommit/v3/request.js b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v3/request.js new file mode 100644 index 0000000..e7d063c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v3/request.js @@ -0,0 +1,20 @@ +const requestV2 = require('../v2/request') + +/** + * OffsetCommit Request (Version: 3) => group_id generation_id member_id retention_time [topics] + * group_id => STRING + * generation_id => INT32 + * member_id => STRING + * retention_time => INT64 + * topics => topic [partitions] + * topic => STRING + * partitions => partition offset metadata + * partition => INT32 + * offset => INT64 + * metadata => NULLABLE_STRING + */ + +module.exports = ({ groupId, groupGenerationId, memberId, retentionTime, topics }) => + Object.assign(requestV2({ groupId, groupGenerationId, memberId, retentionTime, topics }), { + apiVersion: 3, + }) diff --git a/node_modules/kafkajs/src/protocol/requests/offsetCommit/v3/response.js b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v3/response.js new file mode 100644 index 0000000..7cafccf --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v3/response.js @@ -0,0 +1,35 @@ +const Decoder = require('../../../decoder') +const { parse: parseV0 } = require('../v0/response') + +/** + * OffsetCommit Response (Version: 3) => throttle_time_ms [responses] + * throttle_time_ms => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code + * partition => INT32 + * error_code => INT16 + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + throttleTime: decoder.readInt32(), + responses: decoder.readArray(decodeResponses), + } +} + +const decodeResponses = decoder => ({ + topic: decoder.readString(), + partitions: decoder.readArray(decodePartitions), +}) + +const decodePartitions = decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), +}) + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetCommit/v4/request.js b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v4/request.js new file mode 100644 index 0000000..7e86063 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v4/request.js @@ -0,0 +1,20 @@ +const requestV3 = require('../v3/request') + +/** + * OffsetCommit Request (Version: 4) => group_id generation_id member_id retention_time [topics] + * group_id => STRING + * generation_id => INT32 + * member_id => STRING + * retention_time => INT64 + * topics => topic [partitions] + * topic => STRING + * partitions => partition offset metadata + * partition => INT32 + * offset => INT64 + * metadata => NULLABLE_STRING + */ + +module.exports = ({ groupId, groupGenerationId, memberId, retentionTime, topics }) => + Object.assign(requestV3({ groupId, groupGenerationId, memberId, retentionTime, topics }), { + apiVersion: 4, + }) diff --git a/node_modules/kafkajs/src/protocol/requests/offsetCommit/v4/response.js b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v4/response.js new file mode 100644 index 0000000..7ef7291 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v4/response.js @@ -0,0 +1,29 @@ +const { parse, decode: decodeV3 } = require('../v3/response') + +/** + * Starting in version 4, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * OffsetCommit Response (Version: 4) => throttle_time_ms [responses] + * throttle_time_ms => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code + * partition => INT32 + * error_code => INT16 + */ + +const decode = async rawData => { + const decoded = await decodeV3(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetCommit/v5/request.js b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v5/request.js new file mode 100644 index 0000000..7cabe55 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v5/request.js @@ -0,0 +1,41 @@ +const Encoder = require('../../../encoder') +const { OffsetCommit: apiKey } = require('../../apiKeys') + +/** + * Version 5 removes retention_time, as this is controlled by a broker setting + * + * OffsetCommit Request (Version: 4) => group_id generation_id member_id [topics] + * group_id => STRING + * generation_id => INT32 + * member_id => STRING + * topics => topic [partitions] + * topic => STRING + * partitions => partition offset metadata + * partition => INT32 + * offset => INT64 + * metadata => NULLABLE_STRING + */ + +module.exports = ({ groupId, groupGenerationId, memberId, topics }) => ({ + apiKey, + apiVersion: 5, + apiName: 'OffsetCommit', + encode: async () => { + return new Encoder() + .writeString(groupId) + .writeInt32(groupGenerationId) + .writeString(memberId) + .writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition, offset, metadata = null }) => { + return new Encoder() + .writeInt32(partition) + .writeInt64(offset) + .writeString(metadata) +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetCommit/v5/response.js b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v5/response.js new file mode 100644 index 0000000..c2c8fa1 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetCommit/v5/response.js @@ -0,0 +1,15 @@ +const { parse, decode } = require('../v4/response') + +/** + * OffsetCommit Response (Version: 5) => throttle_time_ms [responses] + * throttle_time_ms => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code + * partition => INT32 + * error_code => INT16 + */ +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetFetch/index.js b/node_modules/kafkajs/src/protocol/requests/offsetFetch/index.js new file mode 100644 index 0000000..98abee2 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetFetch/index.js @@ -0,0 +1,27 @@ +const versions = { + 1: ({ groupId, topics }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ groupId, topics }), response } + }, + 2: ({ groupId, topics }) => { + const request = require('./v2/request') + const response = require('./v2/response') + return { request: request({ groupId, topics }), response } + }, + 3: ({ groupId, topics }) => { + const request = require('./v3/request') + const response = require('./v3/response') + return { request: request({ groupId, topics }), response } + }, + 4: ({ groupId, topics }) => { + const request = require('./v4/request') + const response = require('./v4/response') + return { request: request({ groupId, topics }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetFetch/v1/request.js b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v1/request.js new file mode 100644 index 0000000..6909956 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v1/request.js @@ -0,0 +1,28 @@ +const Encoder = require('../../../encoder') +const { OffsetFetch: apiKey } = require('../../apiKeys') + +/** + * OffsetFetch Request (Version: 1) => group_id [topics] + * group_id => STRING + * topics => topic [partitions] + * topic => STRING + * partitions => partition + * partition => INT32 + */ + +module.exports = ({ groupId, topics }) => ({ + apiKey, + apiVersion: 1, + apiName: 'OffsetFetch', + encode: async () => { + return new Encoder().writeString(groupId).writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition }) => { + return new Encoder().writeInt32(partition) +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetFetch/v1/response.js b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v1/response.js new file mode 100644 index 0000000..31684b5 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v1/response.js @@ -0,0 +1,49 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * OffsetFetch Response (Version: 1) => [responses] + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition offset metadata error_code + * partition => INT32 + * offset => INT64 + * metadata => NULLABLE_STRING + * error_code => INT16 + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + responses: decoder.readArray(decodeResponses), + } +} + +const decodeResponses = decoder => ({ + topic: decoder.readString(), + partitions: decoder.readArray(decodePartitions), +}) + +const decodePartitions = decoder => ({ + partition: decoder.readInt32(), + offset: decoder.readInt64().toString(), + metadata: decoder.readString(), + errorCode: decoder.readInt16(), +}) + +const parse = async data => { + const partitionsWithError = data.responses.flatMap(response => + response.partitions.filter(partition => failure(partition.errorCode)) + ) + const partitionWithError = partitionsWithError[0] + if (partitionWithError) { + throw createErrorFromCode(partitionWithError.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetFetch/v2/request.js b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v2/request.js new file mode 100644 index 0000000..e958730 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v2/request.js @@ -0,0 +1,13 @@ +const requestV1 = require('../v1/request') + +/** + * OffsetFetch Request (Version: 2) => group_id [topics] + * group_id => STRING + * topics => topic [partitions] + * topic => STRING + * partitions => partition + * partition => INT32 + */ + +module.exports = ({ groupId, topics }) => + Object.assign(requestV1({ groupId, topics }), { apiVersion: 2 }) diff --git a/node_modules/kafkajs/src/protocol/requests/offsetFetch/v2/response.js b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v2/response.js new file mode 100644 index 0000000..a772d9d --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v2/response.js @@ -0,0 +1,55 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * OffsetFetch Response (Version: 2) => [responses] error_code + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition offset metadata error_code + * partition => INT32 + * offset => INT64 + * metadata => NULLABLE_STRING + * error_code => INT16 + * error_code => INT16 + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + responses: decoder.readArray(decodeResponses), + errorCode: decoder.readInt16(), + } +} + +const decodeResponses = decoder => ({ + topic: decoder.readString(), + partitions: decoder.readArray(decodePartitions), +}) + +const decodePartitions = decoder => ({ + partition: decoder.readInt32(), + offset: decoder.readInt64().toString(), + metadata: decoder.readString(), + errorCode: decoder.readInt16(), +}) + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + const partitionsWithError = data.responses.flatMap(response => + response.partitions.filter(partition => failure(partition.errorCode)) + ) + const partitionWithError = partitionsWithError[0] + if (partitionWithError) { + throw createErrorFromCode(partitionWithError.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetFetch/v3/request.js b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v3/request.js new file mode 100644 index 0000000..d4dadec --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v3/request.js @@ -0,0 +1,28 @@ +const Encoder = require('../../../encoder') +const { OffsetFetch: apiKey } = require('../../apiKeys') + +/** + * OffsetFetch Request (Version: 3) => group_id [topics] + * group_id => STRING + * topics => topic [partitions] + * topic => STRING + * partitions => partition + * partition => INT32 + */ + +module.exports = ({ groupId, topics }) => ({ + apiKey, + apiVersion: 3, + apiName: 'OffsetFetch', + encode: async () => { + return new Encoder().writeString(groupId).writeNullableArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition }) => { + return new Encoder().writeInt32(partition) +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetFetch/v3/response.js b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v3/response.js new file mode 100644 index 0000000..09a8baf --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v3/response.js @@ -0,0 +1,41 @@ +const Decoder = require('../../../decoder') +const { parse: parseV2 } = require('../v2/response') + +/** + * OffsetFetch Response (Version: 3) => throttle_time_ms [responses] error_code + * throttle_time_ms => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition offset metadata error_code + * partition => INT32 + * offset => INT64 + * metadata => NULLABLE_STRING + * error_code => INT16 + * error_code => INT16 + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + return { + throttleTime: decoder.readInt32(), + responses: decoder.readArray(decodeResponses), + errorCode: decoder.readInt16(), + } +} + +const decodeResponses = decoder => ({ + topic: decoder.readString(), + partitions: decoder.readArray(decodePartitions), +}) + +const decodePartitions = decoder => ({ + partition: decoder.readInt32(), + offset: decoder.readInt64().toString(), + metadata: decoder.readString(), + errorCode: decoder.readInt16(), +}) + +module.exports = { + decode, + parse: parseV2, +} diff --git a/node_modules/kafkajs/src/protocol/requests/offsetFetch/v4/request.js b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v4/request.js new file mode 100644 index 0000000..66e6603 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v4/request.js @@ -0,0 +1,13 @@ +const requestV3 = require('../v3/request') + +/** + * OffsetFetch Request (Version: 4) => group_id [topics] + * group_id => STRING + * topics => topic [partitions] + * topic => STRING + * partitions => partition + * partition => INT32 + */ + +module.exports = ({ groupId, topics }) => + Object.assign(requestV3({ groupId, topics }), { apiVersion: 4 }) diff --git a/node_modules/kafkajs/src/protocol/requests/offsetFetch/v4/response.js b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v4/response.js new file mode 100644 index 0000000..02ae3da --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/offsetFetch/v4/response.js @@ -0,0 +1,32 @@ +const { parse, decode: decodeV3 } = require('../v3/response') + +/** + * Starting in version 4, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * OffsetFetch Response (Version: 4) => throttle_time_ms [responses] error_code + * throttle_time_ms => INT32 + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition offset metadata error_code + * partition => INT32 + * offset => INT64 + * metadata => NULLABLE_STRING + * error_code => INT16 + * error_code => INT16 + */ + +const decode = async rawData => { + const decoded = await decodeV3(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/produce/index.js b/node_modules/kafkajs/src/protocol/requests/produce/index.js new file mode 100644 index 0000000..b453607 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/index.js @@ -0,0 +1,102 @@ +const versions = { + 0: ({ acks, timeout, topicData }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ acks, timeout, topicData }), response } + }, + 1: ({ acks, timeout, topicData }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ acks, timeout, topicData }), response } + }, + 2: ({ acks, timeout, topicData, compression }) => { + const request = require('./v2/request') + const response = require('./v2/response') + return { request: request({ acks, timeout, compression, topicData }), response } + }, + 3: ({ acks, timeout, compression, topicData, transactionalId, producerId, producerEpoch }) => { + const request = require('./v3/request') + const response = require('./v3/response') + return { + request: request({ + acks, + timeout, + compression, + topicData, + transactionalId, + producerId, + producerEpoch, + }), + response, + } + }, + 4: ({ acks, timeout, compression, topicData, transactionalId, producerId, producerEpoch }) => { + const request = require('./v4/request') + const response = require('./v4/response') + return { + request: request({ + acks, + timeout, + compression, + topicData, + transactionalId, + producerId, + producerEpoch, + }), + response, + } + }, + 5: ({ acks, timeout, compression, topicData, transactionalId, producerId, producerEpoch }) => { + const request = require('./v5/request') + const response = require('./v5/response') + return { + request: request({ + acks, + timeout, + compression, + topicData, + transactionalId, + producerId, + producerEpoch, + }), + response, + } + }, + 6: ({ acks, timeout, compression, topicData, transactionalId, producerId, producerEpoch }) => { + const request = require('./v6/request') + const response = require('./v6/response') + return { + request: request({ + acks, + timeout, + compression, + topicData, + transactionalId, + producerId, + producerEpoch, + }), + response, + } + }, + 7: ({ acks, timeout, compression, topicData, transactionalId, producerId, producerEpoch }) => { + const request = require('./v7/request') + const response = require('./v7/response') + return { + request: request({ + acks, + timeout, + compression, + topicData, + transactionalId, + producerId, + producerEpoch, + }), + response, + } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v0/request.js b/node_modules/kafkajs/src/protocol/requests/produce/v0/request.js new file mode 100644 index 0000000..460e33c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v0/request.js @@ -0,0 +1,86 @@ +const Encoder = require('../../../encoder') +const { Produce: apiKey } = require('../../apiKeys') +const MessageSet = require('../../../messageSet') + +/** + * Produce Request (Version: 0) => acks timeout [topic_data] + * acks => INT16 + * timeout => INT32 + * topic_data => topic [data] + * topic => STRING + * data => partition record_set record_set_size + * partition => INT32 + * record_set_size => INT32 + * record_set => RECORDS + */ + +/** + * MessageV0: + * { + * key: bytes, + * value: bytes + * } + * + * MessageSet: + * [ + * { key: "", value: "" }, + * { key: "", value: "" }, + * ] + * + * TopicData: + * [ + * { + * topic: 'name1', + * partitions: [ + * { + * partition: 0, + * messages: [] + * } + * ] + * } + * ] + */ + +/** + * @param acks {Integer} This field indicates how many acknowledgements the servers should receive before + * responding to the request. If it is 0 the server will not send any response + * (this is the only case where the server will not reply to a request). If it is 1, + * the server will wait the data is written to the local log before sending a response. + * If it is -1 the server will block until the message is committed by all in sync replicas + * before sending a response. + * + * @param timeout {Integer} This provides a maximum time in milliseconds the server can await the receipt of the number + * of acknowledgements in RequiredAcks. The timeout is not an exact limit on the request time + * for a few reasons: + * (1) it does not include network latency, + * (2) the timer begins at the beginning of the processing of this request so if many requests are + * queued due to server overload that wait time will not be included, + * (3) we will not terminate a local write so if the local write time exceeds this timeout it will not + * be respected. To get a hard timeout of this type the client should use the socket timeout. + * + * @param topicData {Array} + */ +module.exports = ({ acks, timeout, topicData }) => ({ + apiKey, + apiVersion: 0, + apiName: 'Produce', + expectResponse: () => acks !== 0, + encode: async () => { + return new Encoder() + .writeInt16(acks) + .writeInt32(timeout) + .writeArray(topicData.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartitions)) +} + +const encodePartitions = ({ partition, messages }) => { + const messageSet = MessageSet({ messageVersion: 0, entries: messages }) + return new Encoder() + .writeInt32(partition) + .writeInt32(messageSet.size()) + .writeEncoder(messageSet) +} diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v0/response.js b/node_modules/kafkajs/src/protocol/requests/produce/v0/response.js new file mode 100644 index 0000000..7388d0f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v0/response.js @@ -0,0 +1,47 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * v0 + * ProduceResponse => [TopicName [Partition ErrorCode Offset]] + * TopicName => string + * Partition => int32 + * ErrorCode => int16 + * Offset => int64 + */ + +const partition = decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + offset: decoder.readInt64().toString(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const topics = decoder.readArray(decoder => ({ + topicName: decoder.readString(), + partitions: decoder.readArray(partition), + })) + + return { + topics, + } +} + +const parse = async data => { + const errors = data.topics.flatMap(topic => { + return topic.partitions.filter(partition => failure(partition.errorCode)) + }) + + if (errors.length > 0) { + const { errorCode } = errors[0] + throw createErrorFromCode(errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v1/request.js b/node_modules/kafkajs/src/protocol/requests/produce/v1/request.js new file mode 100644 index 0000000..e390fa4 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v1/request.js @@ -0,0 +1,8 @@ +const requestV0 = require('../v0/request') + +// Produce Request on or after v1 indicates the client can parse the quota throttle time +// in the Produce Response. + +module.exports = ({ acks, timeout, topicData }) => { + return Object.assign(requestV0({ acks, timeout, topicData }), { apiVersion: 1 }) +} diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v1/response.js b/node_modules/kafkajs/src/protocol/requests/produce/v1/response.js new file mode 100644 index 0000000..7350dc3 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v1/response.js @@ -0,0 +1,38 @@ +const Decoder = require('../../../decoder') +const { parse: parseV0 } = require('../v0/response') + +/** + * v1 (supported in 0.9.0 or later) + * ProduceResponse => [TopicName [Partition ErrorCode Offset]] ThrottleTime + * TopicName => string + * Partition => int32 + * ErrorCode => int16 + * Offset => int64 + * ThrottleTime => int32 + */ + +const partition = decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + offset: decoder.readInt64().toString(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const topics = decoder.readArray(decoder => ({ + topicName: decoder.readString(), + partitions: decoder.readArray(partition), + })) + + const throttleTime = decoder.readInt32() + + return { + topics, + throttleTime, + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v2/request.js b/node_modules/kafkajs/src/protocol/requests/produce/v2/request.js new file mode 100644 index 0000000..486cb6b --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v2/request.js @@ -0,0 +1,66 @@ +const Encoder = require('../../../encoder') +const { Produce: apiKey } = require('../../apiKeys') +const MessageSet = require('../../../messageSet') +const { Types, lookupCodec } = require('../../../message/compression') + +// Produce Request on or after v2 indicates the client can parse the timestamp field +// in the produce Response. + +module.exports = ({ acks, timeout, compression = Types.None, topicData }) => ({ + apiKey, + apiVersion: 2, + apiName: 'Produce', + expectResponse: () => acks !== 0, + encode: async () => { + const encodeTopic = topicEncoder(compression) + const encodedTopicData = [] + + for (const data of topicData) { + encodedTopicData.push(await encodeTopic(data)) + } + + return new Encoder() + .writeInt16(acks) + .writeInt32(timeout) + .writeArray(encodedTopicData) + }, +}) + +const topicEncoder = compression => { + const encodePartitions = partitionsEncoder(compression) + + return async ({ topic, partitions }) => { + const encodedPartitions = [] + + for (const data of partitions) { + encodedPartitions.push(await encodePartitions(data)) + } + + return new Encoder().writeString(topic).writeArray(encodedPartitions) + } +} + +const partitionsEncoder = compression => async ({ partition, messages }) => { + const messageSet = MessageSet({ messageVersion: 1, compression, entries: messages }) + + if (compression === Types.None) { + return new Encoder() + .writeInt32(partition) + .writeInt32(messageSet.size()) + .writeEncoder(messageSet) + } + + const timestamp = messages[0].timestamp || Date.now() + + const codec = lookupCodec(compression) + const compressedValue = await codec.compress(messageSet) + const compressedMessageSet = MessageSet({ + messageVersion: 1, + entries: [{ compression, timestamp, value: compressedValue }], + }) + + return new Encoder() + .writeInt32(partition) + .writeInt32(compressedMessageSet.size()) + .writeEncoder(compressedMessageSet) +} diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v2/response.js b/node_modules/kafkajs/src/protocol/requests/produce/v2/response.js new file mode 100644 index 0000000..05e1562 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v2/response.js @@ -0,0 +1,40 @@ +const Decoder = require('../../../decoder') +const { parse: parseV0 } = require('../v0/response') + +/** + * v2 (supported in 0.10.0 or later) + * ProduceResponse => [TopicName [Partition ErrorCode Offset Timestamp]] ThrottleTime + * TopicName => string + * Partition => int32 + * ErrorCode => int16 + * Offset => int64 + * Timestamp => int64 + * ThrottleTime => int32 + */ + +const partition = decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + offset: decoder.readInt64().toString(), + timestamp: decoder.readInt64().toString(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const topics = decoder.readArray(decoder => ({ + topicName: decoder.readString(), + partitions: decoder.readArray(partition), + })) + + const throttleTime = decoder.readInt32() + + return { + topics, + throttleTime, + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v3/request.js b/node_modules/kafkajs/src/protocol/requests/produce/v3/request.js new file mode 100644 index 0000000..9b81e0f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v3/request.js @@ -0,0 +1,119 @@ +const Long = require('../../../../utils/long') +const Encoder = require('../../../encoder') +const { Produce: apiKey } = require('../../apiKeys') +const { Types } = require('../../../message/compression') +const Record = require('../../../recordBatch/record/v0') +const { RecordBatch } = require('../../../recordBatch/v0') + +/** + * Produce Request (Version: 3) => transactional_id acks timeout [topic_data] + * transactional_id => NULLABLE_STRING + * acks => INT16 + * timeout => INT32 + * topic_data => topic [data] + * topic => STRING + * data => partition record_set + * partition => INT32 + * record_set => RECORDS + */ + +/** + * @param [transactionalId=null] {String} The transactional id or null if the producer is not transactional + * @param acks {Integer} See producer request v0 + * @param timeout {Integer} See producer request v0 + * @param [compression=CompressionTypes.None] {CompressionTypes} + * @param topicData {Array} + */ +module.exports = ({ + acks, + timeout, + transactionalId = null, + producerId = Long.fromInt(-1), + producerEpoch = 0, + compression = Types.None, + topicData, +}) => ({ + apiKey, + apiVersion: 3, + apiName: 'Produce', + expectResponse: () => acks !== 0, + encode: async () => { + const encodeTopic = topicEncoder(compression) + const encodedTopicData = [] + + for (const data of topicData) { + encodedTopicData.push( + await encodeTopic({ ...data, transactionalId, producerId, producerEpoch }) + ) + } + + return new Encoder() + .writeString(transactionalId) + .writeInt16(acks) + .writeInt32(timeout) + .writeArray(encodedTopicData) + }, +}) + +const topicEncoder = compression => async ({ + topic, + partitions, + transactionalId, + producerId, + producerEpoch, +}) => { + const encodePartitions = partitionsEncoder(compression) + const encodedPartitions = [] + + for (const data of partitions) { + encodedPartitions.push( + await encodePartitions({ ...data, transactionalId, producerId, producerEpoch }) + ) + } + + return new Encoder().writeString(topic).writeArray(encodedPartitions) +} + +const partitionsEncoder = compression => async ({ + partition, + messages, + transactionalId, + firstSequence, + producerId, + producerEpoch, +}) => { + const dateNow = Date.now() + const messageTimestamps = messages + .map(m => m.timestamp) + .filter(timestamp => timestamp != null) + .sort() + + const timestamps = messageTimestamps.length === 0 ? [dateNow] : messageTimestamps + const firstTimestamp = timestamps[0] + const maxTimestamp = timestamps[timestamps.length - 1] + + const records = messages.map((message, i) => + Record({ + ...message, + offsetDelta: i, + timestampDelta: (message.timestamp || dateNow) - firstTimestamp, + }) + ) + + const recordBatch = await RecordBatch({ + compression, + records, + firstTimestamp, + maxTimestamp, + producerId, + producerEpoch, + firstSequence, + transactional: !!transactionalId, + lastOffsetDelta: records.length - 1, + }) + + return new Encoder() + .writeInt32(partition) + .writeInt32(recordBatch.size()) + .writeEncoder(recordBatch) +} diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v3/response.js b/node_modules/kafkajs/src/protocol/requests/produce/v3/response.js new file mode 100644 index 0000000..bd5170f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v3/response.js @@ -0,0 +1,54 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * Produce Response (Version: 3) => [responses] throttle_time_ms + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code base_offset log_append_time + * partition => INT32 + * error_code => INT16 + * base_offset => INT64 + * log_append_time => INT64 + * throttle_time_ms => INT32 + */ + +const partition = decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + baseOffset: decoder.readInt64().toString(), + logAppendTime: decoder.readInt64().toString(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const topics = decoder.readArray(decoder => ({ + topicName: decoder.readString(), + partitions: decoder.readArray(partition), + })) + + const throttleTime = decoder.readInt32() + + return { + topics, + throttleTime, + } +} + +const parse = async data => { + const errors = data.topics.flatMap(response => { + return response.partitions.filter(partition => failure(partition.errorCode)) + }) + + if (errors.length > 0) { + const { errorCode } = errors[0] + throw createErrorFromCode(errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v4/request.js b/node_modules/kafkajs/src/protocol/requests/produce/v4/request.js new file mode 100644 index 0000000..fc271e0 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v4/request.js @@ -0,0 +1,35 @@ +const requestV3 = require('../v3/request') + +/** + * Produce Request (Version: 4) => transactional_id acks timeout [topic_data] + * transactional_id => NULLABLE_STRING + * acks => INT16 + * timeout => INT32 + * topic_data => topic [data] + * topic => STRING + * data => partition record_set + * partition => INT32 + * record_set => RECORDS + */ + +module.exports = ({ + acks, + timeout, + transactionalId, + producerId, + producerEpoch, + compression, + topicData, +}) => + Object.assign( + requestV3({ + acks, + timeout, + transactionalId, + producerId, + producerEpoch, + compression, + topicData, + }), + { apiVersion: 4 } + ) diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v4/response.js b/node_modules/kafkajs/src/protocol/requests/produce/v4/response.js new file mode 100644 index 0000000..db11dc1 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v4/response.js @@ -0,0 +1,18 @@ +const { decode, parse } = require('../v3/response') + +/** + * Produce Response (Version: 4) => [responses] throttle_time_ms + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code base_offset log_append_time + * partition => INT32 + * error_code => INT16 + * base_offset => INT64 + * log_append_time => INT64 + * throttle_time_ms => INT32 + */ + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v5/request.js b/node_modules/kafkajs/src/protocol/requests/produce/v5/request.js new file mode 100644 index 0000000..adb8533 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v5/request.js @@ -0,0 +1,35 @@ +const requestV3 = require('../v3/request') + +/** + * Produce Request (Version: 5) => transactional_id acks timeout [topic_data] + * transactional_id => NULLABLE_STRING + * acks => INT16 + * timeout => INT32 + * topic_data => topic [data] + * topic => STRING + * data => partition record_set + * partition => INT32 + * record_set => RECORDS + */ + +module.exports = ({ + acks, + timeout, + transactionalId, + producerId, + producerEpoch, + compression, + topicData, +}) => + Object.assign( + requestV3({ + acks, + timeout, + transactionalId, + producerId, + producerEpoch, + compression, + topicData, + }), + { apiVersion: 5 } + ) diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v5/response.js b/node_modules/kafkajs/src/protocol/requests/produce/v5/response.js new file mode 100644 index 0000000..2d5c494 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v5/response.js @@ -0,0 +1,43 @@ +const Decoder = require('../../../decoder') +const { parse: parseV3 } = require('../v3/response') + +/** + * Produce Response (Version: 5) => [responses] throttle_time_ms + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code base_offset log_append_time log_start_offset + * partition => INT32 + * error_code => INT16 + * base_offset => INT64 + * log_append_time => INT64 + * log_start_offset => INT64 + * throttle_time_ms => INT32 + */ + +const partition = decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), + baseOffset: decoder.readInt64().toString(), + logAppendTime: decoder.readInt64().toString(), + logStartOffset: decoder.readInt64().toString(), +}) + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const topics = decoder.readArray(decoder => ({ + topicName: decoder.readString(), + partitions: decoder.readArray(partition), + })) + + const throttleTime = decoder.readInt32() + + return { + topics, + throttleTime, + } +} + +module.exports = { + decode, + parse: parseV3, +} diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v6/request.js b/node_modules/kafkajs/src/protocol/requests/produce/v6/request.js new file mode 100644 index 0000000..c82cacb --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v6/request.js @@ -0,0 +1,38 @@ +const requestV5 = require('../v5/request') + +/** + * The version number is bumped to indicate that on quota violation brokers send out responses before throttling. + * @see https://github.com/apache/kafka/blob/9c8f75c4b624084c954b4da69f092211a9ac4689/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java#L113-L117 + * + * Produce Request (Version: 6) => transactional_id acks timeout [topic_data] + * transactional_id => NULLABLE_STRING + * acks => INT16 + * timeout => INT32 + * topic_data => topic [data] + * topic => STRING + * data => partition record_set + * partition => INT32 + * record_set => RECORDS + */ + +module.exports = ({ + acks, + timeout, + transactionalId, + producerId, + producerEpoch, + compression, + topicData, +}) => + Object.assign( + requestV5({ + acks, + timeout, + transactionalId, + producerId, + producerEpoch, + compression, + topicData, + }), + { apiVersion: 6 } + ) diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v6/response.js b/node_modules/kafkajs/src/protocol/requests/produce/v6/response.js new file mode 100644 index 0000000..4dffc40 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v6/response.js @@ -0,0 +1,32 @@ +const { parse, decode: decodeV5 } = require('../v5/response') + +/** + * The version number is bumped to indicate that on quota violation brokers send out responses before throttling. + * @see https://github.com/apache/kafka/blob/9c8f75c4b624084c954b4da69f092211a9ac4689/clients/src/main/java/org/apache/kafka/common/requests/ProduceResponse.java#L152-L156 + * + * Produce Response (Version: 6) => [responses] throttle_time_ms + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code base_offset log_append_time log_start_offset + * partition => INT32 + * error_code => INT16 + * base_offset => INT64 + * log_append_time => INT64 + * log_start_offset => INT64 + * throttle_time_ms => INT32 + */ + +const decode = async rawData => { + const decoded = await decodeV5(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v7/request.js b/node_modules/kafkajs/src/protocol/requests/produce/v7/request.js new file mode 100644 index 0000000..9375ade --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v7/request.js @@ -0,0 +1,38 @@ +const requestV6 = require('../v6/request') + +/** + * V7 indicates ZStandard capability (see KIP-110) + * @see https://github.com/apache/kafka/blob/9c8f75c4b624084c954b4da69f092211a9ac4689/clients/src/main/java/org/apache/kafka/common/requests/ProduceRequest.java#L118-L121 + * + * Produce Request (Version: 7) => transactional_id acks timeout [topic_data] + * transactional_id => NULLABLE_STRING + * acks => INT16 + * timeout => INT32 + * topic_data => topic [data] + * topic => STRING + * data => partition record_set + * partition => INT32 + * record_set => RECORDS + */ + +module.exports = ({ + acks, + timeout, + transactionalId, + producerId, + producerEpoch, + compression, + topicData, +}) => + Object.assign( + requestV6({ + acks, + timeout, + transactionalId, + producerId, + producerEpoch, + compression, + topicData, + }), + { apiVersion: 7 } + ) diff --git a/node_modules/kafkajs/src/protocol/requests/produce/v7/response.js b/node_modules/kafkajs/src/protocol/requests/produce/v7/response.js new file mode 100644 index 0000000..b958416 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/produce/v7/response.js @@ -0,0 +1,19 @@ +const { decode, parse } = require('../v6/response') + +/** + * Produce Response (Version: 7) => [responses] throttle_time_ms + * responses => topic [partition_responses] + * topic => STRING + * partition_responses => partition error_code base_offset log_append_time log_start_offset + * partition => INT32 + * error_code => INT16 + * base_offset => INT64 + * log_append_time => INT64 + * log_start_offset => INT64 + * throttle_time_ms => INT32 + */ + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/index.js b/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/index.js new file mode 100644 index 0000000..e5d6a33 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/index.js @@ -0,0 +1,17 @@ +const versions = { + 0: ({ authBytes }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ authBytes }), response } + }, + 1: ({ authBytes }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ authBytes }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v0/request.js b/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v0/request.js new file mode 100644 index 0000000..b4c25e8 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v0/request.js @@ -0,0 +1,19 @@ +const Encoder = require('../../../encoder') +const { SaslAuthenticate: apiKey } = require('../../apiKeys') + +/** + * SaslAuthenticate Request (Version: 0) => sasl_auth_bytes + * sasl_auth_bytes => BYTES + */ + +/** + * @param {Buffer} authBytes - SASL authentication bytes from client as defined by the SASL mechanism + */ +module.exports = ({ authBytes }) => ({ + apiKey, + apiVersion: 0, + apiName: 'SaslAuthenticate', + encode: async () => { + return new Encoder().writeBuffer(authBytes) + }, +}) diff --git a/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v0/response.js b/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v0/response.js new file mode 100644 index 0000000..3455fbb --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v0/response.js @@ -0,0 +1,59 @@ +const Decoder = require('../../../decoder') +const Encoder = require('../../../encoder') +const { + failure, + createErrorFromCode, + failIfVersionNotSupported, + errorCodes, +} = require('../../../error') + +const { KafkaJSProtocolError } = require('../../../../errors') +const SASL_AUTHENTICATION_FAILED = 58 +const protocolAuthError = errorCodes.find(e => e.code === SASL_AUTHENTICATION_FAILED) + +/** + * SaslAuthenticate Response (Version: 0) => error_code error_message sasl_auth_bytes + * error_code => INT16 + * error_message => NULLABLE_STRING + * sasl_auth_bytes => BYTES + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + const errorMessage = decoder.readString() + + // This is necessary to make the response compatible with the original + // mechanism protocols. They expect a byte response, which starts with + // the size + const authBytesEncoder = new Encoder().writeBytes(decoder.readBytes()) + const authBytes = authBytesEncoder.buffer + + return { + errorCode, + errorMessage, + authBytes, + } +} + +const parse = async data => { + if (data.errorCode === SASL_AUTHENTICATION_FAILED && data.errorMessage) { + throw new KafkaJSProtocolError({ + ...protocolAuthError, + message: data.errorMessage, + }) + } + + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v1/request.js b/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v1/request.js new file mode 100644 index 0000000..221a7cc --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v1/request.js @@ -0,0 +1,11 @@ +const requestV0 = require('../v0/request') + +/** + * SaslAuthenticate Request (Version: 1) => sasl_auth_bytes + * sasl_auth_bytes => BYTES + */ + +/** + * @param {Buffer} authBytes - SASL authentication bytes from client as defined by the SASL mechanism + */ +module.exports = ({ authBytes }) => Object.assign(requestV0({ authBytes }), { apiVersion: 1 }) diff --git a/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v1/response.js b/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v1/response.js new file mode 100644 index 0000000..39ebd6b --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/saslAuthenticate/v1/response.js @@ -0,0 +1,37 @@ +const Decoder = require('../../../decoder') +const Encoder = require('../../../encoder') +const { parse: parseV0 } = require('../v0/response') +const { failIfVersionNotSupported } = require('../../../error') + +/** + * SaslAuthenticate Response (Version: 1) => error_code error_message sasl_auth_bytes + * error_code => INT16 + * error_message => NULLABLE_STRING + * sasl_auth_bytes => BYTES + * session_lifetime_ms => INT64 + */ +const decode = async rawData => { + const decoder = new Decoder(rawData) + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + const errorMessage = decoder.readString() + + // This is necessary to make the response compatible with the original + // mechanism protocols. They expect a byte response, which starts with + // the size + const authBytesEncoder = new Encoder().writeBytes(decoder.readBytes()) + const authBytes = authBytesEncoder.buffer + const sessionLifetimeMs = decoder.readInt64().toString() + + return { + errorCode, + errorMessage, + authBytes, + sessionLifetimeMs, + } +} +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/saslHandshake/index.js b/node_modules/kafkajs/src/protocol/requests/saslHandshake/index.js new file mode 100644 index 0000000..0725ffa --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/saslHandshake/index.js @@ -0,0 +1,17 @@ +const versions = { + 0: ({ mechanism }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { request: request({ mechanism }), response } + }, + 1: ({ mechanism }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { request: request({ mechanism }), response } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/saslHandshake/v0/request.js b/node_modules/kafkajs/src/protocol/requests/saslHandshake/v0/request.js new file mode 100644 index 0000000..8ecf6b5 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/saslHandshake/v0/request.js @@ -0,0 +1,17 @@ +const Encoder = require('../../../encoder') +const { SaslHandshake: apiKey } = require('../../apiKeys') + +/** + * SaslHandshake Request (Version: 0) => mechanism + * mechanism => STRING + */ + +/** + * @param {string} mechanism - SASL Mechanism chosen by the client + */ +module.exports = ({ mechanism }) => ({ + apiKey, + apiVersion: 0, + apiName: 'SaslHandshake', + encode: async () => new Encoder().writeString(mechanism), +}) diff --git a/node_modules/kafkajs/src/protocol/requests/saslHandshake/v0/response.js b/node_modules/kafkajs/src/protocol/requests/saslHandshake/v0/response.js new file mode 100644 index 0000000..0399a00 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/saslHandshake/v0/response.js @@ -0,0 +1,33 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode, failIfVersionNotSupported } = require('../../../error') + +/** + * SaslHandshake Response (Version: 0) => error_code [enabled_mechanisms] + * error_code => INT16 + * enabled_mechanisms => STRING + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { + errorCode, + enabledMechanisms: decoder.readArray(decoder => decoder.readString()), + } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/saslHandshake/v1/request.js b/node_modules/kafkajs/src/protocol/requests/saslHandshake/v1/request.js new file mode 100644 index 0000000..240b5e1 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/saslHandshake/v1/request.js @@ -0,0 +1,3 @@ +const requestV0 = require('../v0/request') + +module.exports = ({ mechanism }) => ({ ...requestV0({ mechanism }), apiVersion: 1 }) diff --git a/node_modules/kafkajs/src/protocol/requests/saslHandshake/v1/response.js b/node_modules/kafkajs/src/protocol/requests/saslHandshake/v1/response.js new file mode 100644 index 0000000..9712758 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/saslHandshake/v1/response.js @@ -0,0 +1,6 @@ +const { decode: decodeV0, parse: parseV0 } = require('../v0/response') + +module.exports = { + decode: decodeV0, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/syncGroup/index.js b/node_modules/kafkajs/src/protocol/requests/syncGroup/index.js new file mode 100644 index 0000000..dcf5386 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/syncGroup/index.js @@ -0,0 +1,39 @@ +const versions = { + 0: ({ groupId, generationId, memberId, groupAssignment }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { + request: request({ groupId, generationId, memberId, groupAssignment }), + response, + } + }, + 1: ({ groupId, generationId, memberId, groupAssignment }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { + request: request({ groupId, generationId, memberId, groupAssignment }), + response, + } + }, + 2: ({ groupId, generationId, memberId, groupAssignment }) => { + const request = require('./v2/request') + const response = require('./v2/response') + return { + request: request({ groupId, generationId, memberId, groupAssignment }), + response, + } + }, + 3: ({ groupId, generationId, memberId, groupInstanceId, groupAssignment }) => { + const request = require('./v3/request') + const response = require('./v3/response') + return { + request: request({ groupId, generationId, memberId, groupInstanceId, groupAssignment }), + response, + } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/syncGroup/v0/request.js b/node_modules/kafkajs/src/protocol/requests/syncGroup/v0/request.js new file mode 100644 index 0000000..3afc81d --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/syncGroup/v0/request.js @@ -0,0 +1,29 @@ +const Encoder = require('../../../encoder') +const { SyncGroup: apiKey } = require('../../apiKeys') + +/** + * SyncGroup Request (Version: 0) => group_id generation_id member_id [group_assignment] + * group_id => STRING + * generation_id => INT32 + * member_id => STRING + * group_assignment => member_id member_assignment + * member_id => STRING + * member_assignment => BYTES + */ + +module.exports = ({ groupId, generationId, memberId, groupAssignment }) => ({ + apiKey, + apiVersion: 0, + apiName: 'SyncGroup', + encode: async () => { + return new Encoder() + .writeString(groupId) + .writeInt32(generationId) + .writeString(memberId) + .writeArray(groupAssignment.map(encodeGroupAssignment)) + }, +}) + +const encodeGroupAssignment = ({ memberId, memberAssignment }) => { + return new Encoder().writeString(memberId).writeBytes(memberAssignment) +} diff --git a/node_modules/kafkajs/src/protocol/requests/syncGroup/v0/response.js b/node_modules/kafkajs/src/protocol/requests/syncGroup/v0/response.js new file mode 100644 index 0000000..65b2dc6 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/syncGroup/v0/response.js @@ -0,0 +1,33 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode, failIfVersionNotSupported } = require('../../../error') + +/** + * SyncGroup Response (Version: 0) => error_code member_assignment + * error_code => INT16 + * member_assignment => BYTES + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { + errorCode, + memberAssignment: decoder.readBytes(), + } +} + +const parse = async data => { + if (failure(data.errorCode)) { + throw createErrorFromCode(data.errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/syncGroup/v1/request.js b/node_modules/kafkajs/src/protocol/requests/syncGroup/v1/request.js new file mode 100644 index 0000000..90e46e8 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/syncGroup/v1/request.js @@ -0,0 +1,14 @@ +const requestV0 = require('../v0/request') + +/** + * SyncGroup Request (Version: 1) => group_id generation_id member_id [group_assignment] + * group_id => STRING + * generation_id => INT32 + * member_id => STRING + * group_assignment => member_id member_assignment + * member_id => STRING + * member_assignment => BYTES + */ + +module.exports = ({ groupId, generationId, memberId, groupAssignment }) => + Object.assign(requestV0({ groupId, generationId, memberId, groupAssignment }), { apiVersion: 1 }) diff --git a/node_modules/kafkajs/src/protocol/requests/syncGroup/v1/response.js b/node_modules/kafkajs/src/protocol/requests/syncGroup/v1/response.js new file mode 100644 index 0000000..e96f6a6 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/syncGroup/v1/response.js @@ -0,0 +1,29 @@ +const Decoder = require('../../../decoder') +const { failIfVersionNotSupported } = require('../../../error') +const { parse: parseV0 } = require('../v0/response') + +/** + * SyncGroup Response (Version: 1) => throttle_time_ms error_code member_assignment + * throttle_time_ms => INT32 + * error_code => INT16 + * member_assignment => BYTES + */ + +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const errorCode = decoder.readInt16() + + failIfVersionNotSupported(errorCode) + + return { + throttleTime, + errorCode, + memberAssignment: decoder.readBytes(), + } +} + +module.exports = { + decode, + parse: parseV0, +} diff --git a/node_modules/kafkajs/src/protocol/requests/syncGroup/v2/request.js b/node_modules/kafkajs/src/protocol/requests/syncGroup/v2/request.js new file mode 100644 index 0000000..55e19ac --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/syncGroup/v2/request.js @@ -0,0 +1,14 @@ +const requestV1 = require('../v1/request') + +/** + * SyncGroup Request (Version: 2) => group_id generation_id member_id [group_assignment] + * group_id => STRING + * generation_id => INT32 + * member_id => STRING + * group_assignment => member_id member_assignment + * member_id => STRING + * member_assignment => BYTES + */ + +module.exports = ({ groupId, generationId, memberId, groupAssignment }) => + Object.assign(requestV1({ groupId, generationId, memberId, groupAssignment }), { apiVersion: 2 }) diff --git a/node_modules/kafkajs/src/protocol/requests/syncGroup/v2/response.js b/node_modules/kafkajs/src/protocol/requests/syncGroup/v2/response.js new file mode 100644 index 0000000..2ba0677 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/syncGroup/v2/response.js @@ -0,0 +1,26 @@ +const { parse, decode: decodeV1 } = require('../v1/response') + +/** + * In version 2, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * SyncGroup Response (Version: 2) => throttle_time_ms error_code member_assignment + * throttle_time_ms => INT32 + * error_code => INT16 + * member_assignment => BYTES + */ + +const decode = async rawData => { + const decoded = await decodeV1(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/syncGroup/v3/request.js b/node_modules/kafkajs/src/protocol/requests/syncGroup/v3/request.js new file mode 100644 index 0000000..283bc99 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/syncGroup/v3/request.js @@ -0,0 +1,40 @@ +const Encoder = require('../../../encoder') +const { SyncGroup: apiKey } = require('../../apiKeys') + +/** + * Version 3 adds group_instance_id to indicate member identity across restarts. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-345%3A+Introduce+static+membership+protocol+to+reduce+consumer+rebalances + * + * SyncGroup Request (Version: 3) => group_id generation_id member_id group_instance_id [group_assignment] + * group_id => STRING + * generation_id => INT32 + * member_id => STRING + * group_instance_id => NULLABLE_STRING + * group_assignment => member_id member_assignment + * member_id => STRING + * member_assignment => BYTES + */ + +module.exports = ({ + groupId, + generationId, + memberId, + groupInstanceId = null, + groupAssignment, +}) => ({ + apiKey, + apiVersion: 3, + apiName: 'SyncGroup', + encode: async () => { + return new Encoder() + .writeString(groupId) + .writeInt32(generationId) + .writeString(memberId) + .writeString(groupInstanceId) + .writeArray(groupAssignment.map(encodeGroupAssignment)) + }, +}) + +const encodeGroupAssignment = ({ memberId, memberAssignment }) => { + return new Encoder().writeString(memberId).writeBytes(memberAssignment) +} diff --git a/node_modules/kafkajs/src/protocol/requests/syncGroup/v3/response.js b/node_modules/kafkajs/src/protocol/requests/syncGroup/v3/response.js new file mode 100644 index 0000000..dc4444d --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/syncGroup/v3/response.js @@ -0,0 +1,12 @@ +const { decode, parse } = require('../v2/response') + +/** + * SyncGroup Response (Version: 2) => throttle_time_ms error_code member_assignment + * throttle_time_ms => INT32 + * error_code => INT16 + * member_assignment => BYTES + */ +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/index.js b/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/index.js new file mode 100644 index 0000000..83be40a --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/index.js @@ -0,0 +1,23 @@ +const versions = { + 0: ({ transactionalId, groupId, producerId, producerEpoch, topics }) => { + const request = require('./v0/request') + const response = require('./v0/response') + return { + request: request({ transactionalId, groupId, producerId, producerEpoch, topics }), + response, + } + }, + 1: ({ transactionalId, groupId, producerId, producerEpoch, topics }) => { + const request = require('./v1/request') + const response = require('./v1/response') + return { + request: request({ transactionalId, groupId, producerId, producerEpoch, topics }), + response, + } + }, +} + +module.exports = { + versions: Object.keys(versions), + protocol: ({ version }) => versions[version], +} diff --git a/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v0/request.js b/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v0/request.js new file mode 100644 index 0000000..ea74eed --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v0/request.js @@ -0,0 +1,41 @@ +const Encoder = require('../../../encoder') +const { TxnOffsetCommit: apiKey } = require('../../apiKeys') + +/** + * TxnOffsetCommit Request (Version: 0) => transactional_id group_id producer_id producer_epoch [topics] + * transactional_id => STRING + * group_id => STRING + * producer_id => INT64 + * producer_epoch => INT16 + * topics => topic [partitions] + * topic => STRING + * partitions => partition offset metadata + * partition => INT32 + * offset => INT64 + * metadata => NULLABLE_STRING + */ + +module.exports = ({ transactionalId, groupId, producerId, producerEpoch, topics }) => ({ + apiKey, + apiVersion: 0, + apiName: 'TxnOffsetCommit', + encode: async () => { + return new Encoder() + .writeString(transactionalId) + .writeString(groupId) + .writeInt64(producerId) + .writeInt16(producerEpoch) + .writeArray(topics.map(encodeTopic)) + }, +}) + +const encodeTopic = ({ topic, partitions }) => { + return new Encoder().writeString(topic).writeArray(partitions.map(encodePartition)) +} + +const encodePartition = ({ partition, offset, metadata }) => { + return new Encoder() + .writeInt32(partition) + .writeInt64(offset) + .writeString(metadata) +} diff --git a/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v0/response.js b/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v0/response.js new file mode 100644 index 0000000..c7920c6 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v0/response.js @@ -0,0 +1,51 @@ +const Decoder = require('../../../decoder') +const { failure, createErrorFromCode } = require('../../../error') + +/** + * TxnOffsetCommit Response (Version: 0) => throttle_time_ms [topics] + * throttle_time_ms => INT32 + * topics => topic [partitions] + * topic => STRING + * partitions => partition error_code + * partition => INT32 + * error_code => INT16 + */ +const decode = async rawData => { + const decoder = new Decoder(rawData) + const throttleTime = decoder.readInt32() + const topics = await decoder.readArrayAsync(decodeTopic) + + return { + throttleTime, + topics, + } +} + +const decodeTopic = async decoder => ({ + topic: decoder.readString(), + partitions: await decoder.readArrayAsync(decodePartition), +}) + +const decodePartition = decoder => ({ + partition: decoder.readInt32(), + errorCode: decoder.readInt16(), +}) + +const parse = async data => { + const topicsWithErrors = data.topics + .map(({ partitions }) => ({ + partitionsWithErrors: partitions.filter(({ errorCode }) => failure(errorCode)), + })) + .filter(({ partitionsWithErrors }) => partitionsWithErrors.length) + + if (topicsWithErrors.length > 0) { + throw createErrorFromCode(topicsWithErrors[0].partitionsWithErrors[0].errorCode) + } + + return data +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v1/request.js b/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v1/request.js new file mode 100644 index 0000000..f42b0ba --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v1/request.js @@ -0,0 +1,20 @@ +const requestV0 = require('../v0/request') + +/** + * TxnOffsetCommit Request (Version: 1) => transactional_id group_id producer_id producer_epoch [topics] + * transactional_id => STRING + * group_id => STRING + * producer_id => INT64 + * producer_epoch => INT16 + * topics => topic [partitions] + * topic => STRING + * partitions => partition offset metadata + * partition => INT32 + * offset => INT64 + * metadata => NULLABLE_STRING + */ + +module.exports = ({ transactionalId, groupId, producerId, producerEpoch, topics }) => + Object.assign(requestV0({ transactionalId, groupId, producerId, producerEpoch, topics }), { + apiVersion: 1, + }) diff --git a/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v1/response.js b/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v1/response.js new file mode 100644 index 0000000..6945b1f --- /dev/null +++ b/node_modules/kafkajs/src/protocol/requests/txnOffsetCommit/v1/response.js @@ -0,0 +1,29 @@ +const { parse, decode: decodeV1 } = require('../v0/response') + +/** + * In version 1, on quota violation, brokers send out responses before throttling. + * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-219+-+Improve+quota+communication + * + * TxnOffsetCommit Response (Version: 1) => throttle_time_ms [topics] + * throttle_time_ms => INT32 + * topics => topic [partitions] + * topic => STRING + * partitions => partition error_code + * partition => INT32 + * error_code => INT16 + */ + +const decode = async rawData => { + const decoded = await decodeV1(rawData) + + return { + ...decoded, + throttleTime: 0, + clientSideThrottleTime: decoded.throttleTime, + } +} + +module.exports = { + decode, + parse, +} diff --git a/node_modules/kafkajs/src/protocol/resourcePatternTypes.js b/node_modules/kafkajs/src/protocol/resourcePatternTypes.js new file mode 100644 index 0000000..55fdeb4 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/resourcePatternTypes.js @@ -0,0 +1,46 @@ +// From: +// https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/PatternType.java#L32 + +/** + * @typedef {number} ACLResourcePatternTypes + * + * Enum for ACL Resource Pattern Type + * @readonly + * @enum {ACLResourcePatternTypes} + */ +module.exports = { + /** + * Represents any PatternType which this client cannot understand, perhaps because this client is too old. + */ + UNKNOWN: 0, + /** + * In a filter, matches any resource pattern type. + */ + ANY: 1, + /** + * In a filter, will perform pattern matching. + * + * e.g. Given a filter of {@code ResourcePatternFilter(TOPIC, "payments.received", MATCH)`}, the filter match + * any {@link ResourcePattern} that matches topic 'payments.received'. This might include: + *
    + *
  • A Literal pattern with the same type and name, e.g. {@code ResourcePattern(TOPIC, "payments.received", LITERAL)}
    • + *
  • A Wildcard pattern with the same type, e.g. {@code ResourcePattern(TOPIC, "*", LITERAL)}
    • + *
  • A Prefixed pattern with the same type and where the name is a matching prefix, e.g. {@code ResourcePattern(TOPIC, "payments.", PREFIXED)}
    • + *
+ */ + MATCH: 2, + /** + * A literal resource name. + * + * A literal name defines the full name of a resource, e.g. topic with name 'foo', or group with name 'bob'. + * + * The special wildcard character {@code *} can be used to represent a resource with any name. + */ + LITERAL: 3, + /** + * A prefixed resource name. + * + * A prefixed name defines a prefix for a resource, e.g. topics with names that start with 'foo'. + */ + PREFIXED: 4, +} diff --git a/node_modules/kafkajs/src/protocol/sasl/awsIam/index.js b/node_modules/kafkajs/src/protocol/sasl/awsIam/index.js new file mode 100644 index 0000000..ecb3c9a --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/awsIam/index.js @@ -0,0 +1,4 @@ +module.exports = { + request: require('./request'), + response: require('./response'), +} diff --git a/node_modules/kafkajs/src/protocol/sasl/awsIam/request.js b/node_modules/kafkajs/src/protocol/sasl/awsIam/request.js new file mode 100644 index 0000000..f97149c --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/awsIam/request.js @@ -0,0 +1,11 @@ +const Encoder = require('../../encoder') + +const US_ASCII_NULL_CHAR = '\u0000' + +module.exports = ({ authorizationIdentity, accessKeyId, secretAccessKey, sessionToken = '' }) => ({ + encode: async () => { + return new Encoder().writeBytes( + [authorizationIdentity, accessKeyId, secretAccessKey, sessionToken].join(US_ASCII_NULL_CHAR) + ).buffer + }, +}) diff --git a/node_modules/kafkajs/src/protocol/sasl/awsIam/response.js b/node_modules/kafkajs/src/protocol/sasl/awsIam/response.js new file mode 100644 index 0000000..234bb85 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/awsIam/response.js @@ -0,0 +1,4 @@ +module.exports = { + decode: async () => true, + parse: async () => true, +} diff --git a/node_modules/kafkajs/src/protocol/sasl/oauthBearer/index.js b/node_modules/kafkajs/src/protocol/sasl/oauthBearer/index.js new file mode 100644 index 0000000..ecb3c9a --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/oauthBearer/index.js @@ -0,0 +1,4 @@ +module.exports = { + request: require('./request'), + response: require('./response'), +} diff --git a/node_modules/kafkajs/src/protocol/sasl/oauthBearer/request.js b/node_modules/kafkajs/src/protocol/sasl/oauthBearer/request.js new file mode 100644 index 0000000..3cff0bb --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/oauthBearer/request.js @@ -0,0 +1,62 @@ +/** + * http://www.ietf.org/rfc/rfc5801.txt + * + * See org.apache.kafka.common.security.oauthbearer.internals.OAuthBearerClientInitialResponse + * for official Java client implementation. + * + * The mechanism consists of a message from the client to the server. + * The client sends the "n,"" GS header, followed by the authorizationIdentitty + * prefixed by "a=" (if present), followed by ",", followed by a US-ASCII SOH + * character, followed by "auth=Bearer ", followed by the token value, followed + * by US-ASCII SOH character, followed by SASL extensions in OAuth "friendly" + * format and then closed by two additionals US-ASCII SOH characters. + * + * SASL extensions are optional an must be expressed as key-value pairs in an + * object. Each expression is converted as, the extension entry key, followed + * by "=", followed by extension entry value. Each extension is separated by a + * US-ASCII SOH character. If extensions are not present, their relative part + * in the message, including the US-ASCII SOH character, is omitted. + * + * The client may leave the authorization identity empty to + * indicate that it is the same as the authentication identity. + * + * The server will verify the authentication token and verify that the + * authentication credentials permit the client to login as the authorization + * identity. If both steps succeed, the user is logged in. + */ + +const Encoder = require('../../encoder') + +const SEPARATOR = '\u0001' // SOH - Start Of Header ASCII + +function formatExtensions(extensions) { + let msg = '' + + if (extensions == null) { + return msg + } + + let prefix = '' + for (const k in extensions) { + msg += `${prefix}${k}=${extensions[k]}` + prefix = SEPARATOR + } + + return msg +} + +module.exports = async ({ authorizationIdentity = null }, oauthBearerToken) => { + const authzid = authorizationIdentity == null ? '' : `"a=${authorizationIdentity}` + let ext = formatExtensions(oauthBearerToken.extensions) + if (ext.length > 0) { + ext = `${SEPARATOR}${ext}` + } + + const oauthMsg = `n,${authzid},${SEPARATOR}auth=Bearer ${oauthBearerToken.value}${ext}${SEPARATOR}${SEPARATOR}` + + return { + encode: async () => { + return new Encoder().writeBytes(Buffer.from(oauthMsg)).buffer + }, + } +} diff --git a/node_modules/kafkajs/src/protocol/sasl/oauthBearer/response.js b/node_modules/kafkajs/src/protocol/sasl/oauthBearer/response.js new file mode 100644 index 0000000..234bb85 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/oauthBearer/response.js @@ -0,0 +1,4 @@ +module.exports = { + decode: async () => true, + parse: async () => true, +} diff --git a/node_modules/kafkajs/src/protocol/sasl/plain/index.js b/node_modules/kafkajs/src/protocol/sasl/plain/index.js new file mode 100644 index 0000000..ecb3c9a --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/plain/index.js @@ -0,0 +1,4 @@ +module.exports = { + request: require('./request'), + response: require('./response'), +} diff --git a/node_modules/kafkajs/src/protocol/sasl/plain/request.js b/node_modules/kafkajs/src/protocol/sasl/plain/request.js new file mode 100644 index 0000000..a52c983 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/plain/request.js @@ -0,0 +1,28 @@ +/** + * http://www.ietf.org/rfc/rfc2595.txt + * + * The mechanism consists of a single message from the client to the + * server. The client sends the authorization identity (identity to + * login as), followed by a US-ASCII NUL character, followed by the + * authentication identity (identity whose password will be used), + * followed by a US-ASCII NUL character, followed by the clear-text + * password. The client may leave the authorization identity empty to + * indicate that it is the same as the authentication identity. + * + * The server will verify the authentication identity and password with + * the system authentication database and verify that the authentication + * credentials permit the client to login as the authorization identity. + * If both steps succeed, the user is logged in. + */ + +const Encoder = require('../../encoder') + +const US_ASCII_NULL_CHAR = '\u0000' + +module.exports = ({ authorizationIdentity = null, username, password }) => ({ + encode: async () => { + return new Encoder().writeBytes( + [authorizationIdentity, username, password].join(US_ASCII_NULL_CHAR) + ).buffer + }, +}) diff --git a/node_modules/kafkajs/src/protocol/sasl/plain/response.js b/node_modules/kafkajs/src/protocol/sasl/plain/response.js new file mode 100644 index 0000000..234bb85 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/plain/response.js @@ -0,0 +1,4 @@ +module.exports = { + decode: async () => true, + parse: async () => true, +} diff --git a/node_modules/kafkajs/src/protocol/sasl/scram/finalMessage/request.js b/node_modules/kafkajs/src/protocol/sasl/scram/finalMessage/request.js new file mode 100644 index 0000000..58d40aa --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/scram/finalMessage/request.js @@ -0,0 +1,5 @@ +const Encoder = require('../../../encoder') + +module.exports = ({ finalMessage }) => ({ + encode: async () => new Encoder().writeBytes(finalMessage).buffer, +}) diff --git a/node_modules/kafkajs/src/protocol/sasl/scram/finalMessage/response.js b/node_modules/kafkajs/src/protocol/sasl/scram/finalMessage/response.js new file mode 100644 index 0000000..8b20d98 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/scram/finalMessage/response.js @@ -0,0 +1 @@ +module.exports = require('../firstMessage/response') diff --git a/node_modules/kafkajs/src/protocol/sasl/scram/firstMessage/request.js b/node_modules/kafkajs/src/protocol/sasl/scram/firstMessage/request.js new file mode 100644 index 0000000..865f0e5 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/scram/firstMessage/request.js @@ -0,0 +1,22 @@ +/** + * https://tools.ietf.org/html/rfc5802 + * + * First, the client sends the "client-first-message" containing: + * + * -> a GS2 header consisting of a flag indicating whether channel + * binding is supported-but-not-used, not supported, or used, and an + * optional SASL authorization identity; + * + * -> SCRAM username and a random, unique nonce attributes. + * + * Note that the client's first message will always start with "n", "y", + * or "p"; otherwise, the message is invalid and authentication MUST + * fail. This is important, as it allows for GS2 extensibility (e.g., + * to add support for security layers). + */ + +const Encoder = require('../../../encoder') + +module.exports = ({ clientFirstMessage }) => ({ + encode: async () => new Encoder().writeBytes(clientFirstMessage).buffer, +}) diff --git a/node_modules/kafkajs/src/protocol/sasl/scram/firstMessage/response.js b/node_modules/kafkajs/src/protocol/sasl/scram/firstMessage/response.js new file mode 100644 index 0000000..097f259 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/scram/firstMessage/response.js @@ -0,0 +1,23 @@ +/* eslint no-unused-vars: ["error", { "varsIgnorePattern": "_" }] */ + +const Decoder = require('../../../decoder') + +const ENTRY_REGEX = /^([rsiev])=(.*)$/ + +module.exports = { + decode: async rawData => { + return new Decoder(rawData).readBytes() + }, + parse: async data => { + const processed = data + .toString() + .split(',') + .map(str => { + const [_, key, value] = str.match(ENTRY_REGEX) + return [key, value] + }) + .reduce((obj, entry) => ({ ...obj, [entry[0]]: entry[1] }), {}) + + return { original: data.toString(), ...processed } + }, +} diff --git a/node_modules/kafkajs/src/protocol/sasl/scram/index.js b/node_modules/kafkajs/src/protocol/sasl/scram/index.js new file mode 100644 index 0000000..e5e0af8 --- /dev/null +++ b/node_modules/kafkajs/src/protocol/sasl/scram/index.js @@ -0,0 +1,10 @@ +module.exports = { + firstMessage: { + request: require('./firstMessage/request'), + response: require('./firstMessage/response'), + }, + finalMessage: { + request: require('./finalMessage/request'), + response: require('./finalMessage/response'), + }, +} diff --git a/node_modules/kafkajs/src/protocol/timestampTypes.js b/node_modules/kafkajs/src/protocol/timestampTypes.js new file mode 100644 index 0000000..eb9aafa --- /dev/null +++ b/node_modules/kafkajs/src/protocol/timestampTypes.js @@ -0,0 +1,15 @@ +/** + * Enum for timestamp types + * @readonly + * @enum {TimestampType} + */ +module.exports = { + // Timestamp type is unknown + NO_TIMESTAMP: -1, + + // Timestamp relates to message creation time as set by a Kafka client + CREATE_TIME: 0, + + // Timestamp relates to the time a message was appended to a Kafka log + LOG_APPEND_TIME: 1, +} diff --git a/node_modules/kafkajs/src/retry/defaults.js b/node_modules/kafkajs/src/retry/defaults.js new file mode 100644 index 0000000..71d6462 --- /dev/null +++ b/node_modules/kafkajs/src/retry/defaults.js @@ -0,0 +1,7 @@ +module.exports = { + maxRetryTime: 30 * 1000, + initialRetryTime: 300, + factor: 0.2, // randomization factor + multiplier: 2, // exponential factor + retries: 5, // max retries +} diff --git a/node_modules/kafkajs/src/retry/defaults.test.js b/node_modules/kafkajs/src/retry/defaults.test.js new file mode 100644 index 0000000..0713dde --- /dev/null +++ b/node_modules/kafkajs/src/retry/defaults.test.js @@ -0,0 +1,7 @@ +module.exports = { + maxRetryTime: 1000, + initialRetryTime: 50, + factor: 0.02, // randomization factor + multiplier: 1.5, // exponential factor + retries: 15, // max retries +} diff --git a/node_modules/kafkajs/src/retry/index.js b/node_modules/kafkajs/src/retry/index.js new file mode 100644 index 0000000..17026de --- /dev/null +++ b/node_modules/kafkajs/src/retry/index.js @@ -0,0 +1,77 @@ +const { KafkaJSNumberOfRetriesExceeded, KafkaJSNonRetriableError } = require('../errors') + +const isTestMode = process.env.NODE_ENV === 'test' +const RETRY_DEFAULT = isTestMode ? require('./defaults.test') : require('./defaults') + +const random = (min, max) => { + return Math.random() * (max - min) + min +} + +const randomFromRetryTime = (factor, retryTime) => { + const delta = factor * retryTime + return Math.ceil(random(retryTime - delta, retryTime + delta)) +} + +const UNRECOVERABLE_ERRORS = ['RangeError', 'ReferenceError', 'SyntaxError', 'TypeError'] +const isErrorUnrecoverable = e => UNRECOVERABLE_ERRORS.includes(e.name) +const isErrorRetriable = error => + (error.retriable || error.retriable !== false) && !isErrorUnrecoverable(error) + +const createRetriable = (configs, resolve, reject, fn) => { + let aborted = false + const { factor, multiplier, maxRetryTime, retries } = configs + + const bail = error => { + aborted = true + reject(error || new Error('Aborted')) + } + + const calculateExponentialRetryTime = retryTime => { + return Math.min(randomFromRetryTime(factor, retryTime) * multiplier, maxRetryTime) + } + + const retry = (retryTime, retryCount = 0) => { + if (aborted) return + + const nextRetryTime = calculateExponentialRetryTime(retryTime) + const shouldRetry = retryCount < retries + + const scheduleRetry = () => { + setTimeout(() => retry(nextRetryTime, retryCount + 1), retryTime) + } + + fn(bail, retryCount, retryTime) + .then(resolve) + .catch(e => { + if (isErrorRetriable(e)) { + if (shouldRetry) { + scheduleRetry() + } else { + reject( + new KafkaJSNumberOfRetriesExceeded(e, { retryCount, retryTime, cause: e.cause || e }) + ) + } + } else { + reject(new KafkaJSNonRetriableError(e, { cause: e.cause || e })) + } + }) + } + + return retry +} + +/** + * @typedef {(fn: (bail: (err: Error) => void, retryCount: number, retryTime: number) => any) => Promise>} Retrier + */ + +/** + * @param {import("../../types").RetryOptions} [opts] + * @returns {Retrier} + */ +module.exports = (opts = {}) => fn => { + return new Promise((resolve, reject) => { + const configs = Object.assign({}, RETRY_DEFAULT, opts) + const start = createRetriable(configs, resolve, reject, fn) + start(randomFromRetryTime(configs.factor, configs.initialRetryTime)) + }) +} diff --git a/node_modules/kafkajs/src/utils/arrayDiff.js b/node_modules/kafkajs/src/utils/arrayDiff.js new file mode 100644 index 0000000..2b88ef3 --- /dev/null +++ b/node_modules/kafkajs/src/utils/arrayDiff.js @@ -0,0 +1,14 @@ +module.exports = (a, b) => { + const result = [] + const length = a.length + let i = 0 + + while (i < length) { + if (b.indexOf(a[i]) === -1) { + result.push(a[i]) + } + i += 1 + } + + return result +} diff --git a/node_modules/kafkajs/src/utils/groupBy.js b/node_modules/kafkajs/src/utils/groupBy.js new file mode 100644 index 0000000..a8af00a --- /dev/null +++ b/node_modules/kafkajs/src/utils/groupBy.js @@ -0,0 +1,10 @@ +module.exports = async (array, groupFn) => { + const result = new Map() + + for (const item of array) { + const group = await Promise.resolve(groupFn(item)) + result.set(group, result.has(group) ? [...result.get(group), item] : [item]) + } + + return result +} diff --git a/node_modules/kafkajs/src/utils/lock.js b/node_modules/kafkajs/src/utils/lock.js new file mode 100644 index 0000000..07de4bc --- /dev/null +++ b/node_modules/kafkajs/src/utils/lock.js @@ -0,0 +1,63 @@ +const { format } = require('util') +const { KafkaJSLockTimeout } = require('../errors') + +const PRIVATE = { + LOCKED: Symbol('private:Lock:locked'), + TIMEOUT: Symbol('private:Lock:timeout'), + WAITING: Symbol('private:Lock:waiting'), + TIMEOUT_ERROR_MESSAGE: Symbol('private:Lock:timeoutErrorMessage'), +} + +const TIMEOUT_MESSAGE = 'Timeout while acquiring lock (%d waiting locks)' + +module.exports = class Lock { + constructor({ timeout, description = null } = {}) { + if (typeof timeout !== 'number') { + throw new TypeError(`'timeout' is not a number, received '${typeof timeout}'`) + } + + this[PRIVATE.LOCKED] = false + this[PRIVATE.TIMEOUT] = timeout + this[PRIVATE.WAITING] = new Set() + this[PRIVATE.TIMEOUT_ERROR_MESSAGE] = () => { + const timeoutMessage = format(TIMEOUT_MESSAGE, this[PRIVATE.WAITING].size) + return description ? `${timeoutMessage}: "${description}"` : timeoutMessage + } + } + + async acquire() { + return new Promise((resolve, reject) => { + if (!this[PRIVATE.LOCKED]) { + this[PRIVATE.LOCKED] = true + return resolve() + } + + let timeoutId = null + const tryToAcquire = async () => { + if (!this[PRIVATE.LOCKED]) { + this[PRIVATE.LOCKED] = true + clearTimeout(timeoutId) + this[PRIVATE.WAITING].delete(tryToAcquire) + return resolve() + } + } + + this[PRIVATE.WAITING].add(tryToAcquire) + timeoutId = setTimeout(() => { + // The message should contain the number of waiters _including_ this one + const error = new KafkaJSLockTimeout(this[PRIVATE.TIMEOUT_ERROR_MESSAGE]()) + this[PRIVATE.WAITING].delete(tryToAcquire) + reject(error) + }, this[PRIVATE.TIMEOUT]) + }) + } + + async release() { + this[PRIVATE.LOCKED] = false + const waitingLock = this[PRIVATE.WAITING].values().next().value + + if (waitingLock) { + return waitingLock() + } + } +} diff --git a/node_modules/kafkajs/src/utils/long.js b/node_modules/kafkajs/src/utils/long.js new file mode 100644 index 0000000..e8e8923 --- /dev/null +++ b/node_modules/kafkajs/src/utils/long.js @@ -0,0 +1,343 @@ +/** + * @exports Long + * @class A Long class for representing a 64 bit int (BigInt) + * @param {bigint} value The value of the 64 bit int + * @constructor + */ +class Long { + constructor(value) { + this.value = value + } + + /** + * @function isLong + * @param {*} obj Object + * @returns {boolean} + * @inner + */ + static isLong(obj) { + return typeof obj.value === 'bigint' + } + + /** + * @param {number} value + * @returns {!Long} + * @inner + */ + static fromBits(value) { + return new Long(BigInt(value)) + } + + /** + * @param {number} value + * @returns {!Long} + * @inner + */ + static fromInt(value) { + if (isNaN(value)) return Long.ZERO + + return new Long(BigInt.asIntN(64, BigInt(value))) + } + + /** + * @param {number} value + * @returns {!Long} + * @inner + */ + static fromNumber(value) { + if (isNaN(value)) return Long.ZERO + + return new Long(BigInt(value)) + } + + /** + * @function + * @param {bigint|number|string|Long} val + * @returns {!Long} + * @inner + */ + static fromValue(val) { + if (typeof val === 'number') return this.fromNumber(val) + if (typeof val === 'string') return this.fromString(val) + if (typeof val === 'bigint') return new Long(val) + if (this.isLong(val)) return new Long(BigInt(val.value)) + + return new Long(BigInt(val)) + } + + /** + * @param {string} str + * @returns {!Long} + * @inner + */ + static fromString(str) { + if (str.length === 0) throw Error('empty string') + if (str === 'NaN' || str === 'Infinity' || str === '+Infinity' || str === '-Infinity') + return Long.ZERO + return new Long(BigInt(str)) + } + + /** + * Tests if this Long's value equals zero. + * @returns {boolean} + */ + isZero() { + return this.value === BigInt(0) + } + + /** + * Tests if this Long's value is negative. + * @returns {boolean} + */ + isNegative() { + return this.value < BigInt(0) + } + + /** + * Converts the Long to a string. + * @returns {string} + * @override + */ + toString() { + return String(this.value) + } + + /** + * Converts the Long to the nearest floating-point representation (double, 53-bit mantissa) + * @returns {number} + * @override + */ + toNumber() { + return Number(this.value) + } + + /** + * Converts the Long to a 32 bit integer, assuming it is a 32 bit integer. + * @returns {number} + */ + toInt() { + return Number(BigInt.asIntN(32, this.value)) + } + + /** + * Converts the Long to JSON + * @returns {string} + * @override + */ + toJSON() { + return this.toString() + } + + /** + * Returns this Long with bits shifted to the left by the given amount. + * @param {number|bigint} numBits Number of bits + * @returns {!Long} Shifted bigint + */ + shiftLeft(numBits) { + return new Long(this.value << BigInt(numBits)) + } + + /** + * Returns this Long with bits arithmetically shifted to the right by the given amount. + * @param {number|bigint} numBits Number of bits + * @returns {!Long} Shifted bigint + */ + shiftRight(numBits) { + return new Long(this.value >> BigInt(numBits)) + } + + /** + * Returns the bitwise OR of this Long and the specified. + * @param {bigint|number|string} other Other Long + * @returns {!Long} + */ + or(other) { + if (!Long.isLong(other)) other = Long.fromValue(other) + return Long.fromBits(this.value | other.value) + } + + /** + * Returns the bitwise XOR of this Long and the given one. + * @param {bigint|number|string} other Other Long + * @returns {!Long} + */ + xor(other) { + if (!Long.isLong(other)) other = Long.fromValue(other) + return new Long(this.value ^ other.value) + } + + /** + * Returns the bitwise AND of this Long and the specified. + * @param {bigint|number|string} other Other Long + * @returns {!Long} + */ + and(other) { + if (!Long.isLong(other)) other = Long.fromValue(other) + return new Long(this.value & other.value) + } + + /** + * Returns the bitwise NOT of this Long. + * @returns {!Long} + */ + not() { + return new Long(~this.value) + } + + /** + * Returns this Long with bits logically shifted to the right by the given amount. + * @param {number|bigint} numBits Number of bits + * @returns {!Long} Shifted bigint + */ + shiftRightUnsigned(numBits) { + return new Long(this.value >> BigInt.asUintN(64, BigInt(numBits))) + } + + /** + * Tests if this Long's value equals the specified's. + * @param {bigint|number|string} other Other value + * @returns {boolean} + */ + equals(other) { + if (!Long.isLong(other)) other = Long.fromValue(other) + return this.value === other.value + } + + /** + * Tests if this Long's value is greater than or equal the specified's. + * @param {!Long|number|string} other Other value + * @returns {boolean} + */ + greaterThanOrEqual(other) { + if (!Long.isLong(other)) other = Long.fromValue(other) + return this.value >= other.value + } + + gte(other) { + return this.greaterThanOrEqual(other) + } + + notEquals(other) { + if (!Long.isLong(other)) other = Long.fromValue(other) + return !this.equals(/* validates */ other) + } + + /** + * Returns the sum of this and the specified Long. + * @param {!Long|number|string} addend Addend + * @returns {!Long} Sum + */ + add(addend) { + if (!Long.isLong(addend)) addend = Long.fromValue(addend) + return new Long(this.value + addend.value) + } + + /** + * Returns the difference of this and the specified Long. + * @param {!Long|number|string} subtrahend Subtrahend + * @returns {!Long} Difference + */ + subtract(subtrahend) { + if (!Long.isLong(subtrahend)) subtrahend = Long.fromValue(subtrahend) + return this.add(subtrahend.negate()) + } + + /** + * Returns the product of this and the specified Long. + * @param {!Long|number|string} multiplier Multiplier + * @returns {!Long} Product + */ + multiply(multiplier) { + if (this.isZero()) return Long.ZERO + if (!Long.isLong(multiplier)) multiplier = Long.fromValue(multiplier) + return new Long(this.value * multiplier.value) + } + + /** + * Returns this Long divided by the specified. The result is signed if this Long is signed or + * unsigned if this Long is unsigned. + * @param {!Long|number|string} divisor Divisor + * @returns {!Long} Quotient + */ + divide(divisor) { + if (!Long.isLong(divisor)) divisor = Long.fromValue(divisor) + if (divisor.isZero()) throw Error('division by zero') + return new Long(this.value / divisor.value) + } + + /** + * Compares this Long's value with the specified's. + * @param {!Long|number|string} other Other value + * @returns {number} 0 if they are the same, 1 if the this is greater and -1 + * if the given one is greater + */ + compare(other) { + if (!Long.isLong(other)) other = Long.fromValue(other) + if (this.value === other.value) return 0 + if (this.value > other.value) return 1 + if (other.value > this.value) return -1 + } + + /** + * Tests if this Long's value is less than the specified's. + * @param {!Long|number|string} other Other value + * @returns {boolean} + */ + lessThan(other) { + if (!Long.isLong(other)) other = Long.fromValue(other) + return this.value < other.value + } + + /** + * Negates this Long's value. + * @returns {!Long} Negated Long + */ + negate() { + if (this.equals(Long.MIN_VALUE)) { + return Long.MIN_VALUE + } + return this.not().add(Long.ONE) + } + + /** + * Gets the high 32 bits as a signed integer. + * @returns {number} Signed high bits + */ + getHighBits() { + return Number(BigInt.asIntN(32, this.value >> BigInt(32))) + } + + /** + * Gets the low 32 bits as a signed integer. + * @returns {number} Signed low bits + */ + getLowBits() { + return Number(BigInt.asIntN(32, this.value)) + } +} + +/** + * Minimum signed value. + * @type {bigint} + */ +Long.MIN_VALUE = new Long(BigInt('-9223372036854775808')) + +/** + * Maximum signed value. + * @type {bigint} + */ +Long.MAX_VALUE = new Long(BigInt('9223372036854775807')) + +/** + * Signed zero. + * @type {Long} + */ +Long.ZERO = Long.fromInt(0) + +/** + * Signed one. + * @type {!Long} + */ +Long.ONE = Long.fromInt(1) + +module.exports = Long diff --git a/node_modules/kafkajs/src/utils/mapValues.js b/node_modules/kafkajs/src/utils/mapValues.js new file mode 100644 index 0000000..e46f774 --- /dev/null +++ b/node_modules/kafkajs/src/utils/mapValues.js @@ -0,0 +1,10 @@ +const { entries } = Object + +module.exports = (obj, mapper) => + entries(obj).reduce( + (acc, [key, value]) => ({ + ...acc, + [key]: mapper(value, key), + }), + {} + ) diff --git a/node_modules/kafkajs/src/utils/once.js b/node_modules/kafkajs/src/utils/once.js new file mode 100644 index 0000000..c29ba7b --- /dev/null +++ b/node_modules/kafkajs/src/utils/once.js @@ -0,0 +1,10 @@ +module.exports = fn => { + let called = false + + return (...args) => { + if (!called) { + called = true + return fn(...args) + } + } +} diff --git a/node_modules/kafkajs/src/utils/seq.js b/node_modules/kafkajs/src/utils/seq.js new file mode 100644 index 0000000..e8450f7 --- /dev/null +++ b/node_modules/kafkajs/src/utils/seq.js @@ -0,0 +1,9 @@ +/** + * @param {number} count + * @param {(index: number) => T} [callback] + * @template T + */ +const seq = (count, callback = x => x) => + new Array(count).fill(0).map((_, index) => callback(index)) + +module.exports = seq diff --git a/node_modules/kafkajs/src/utils/sharedPromiseTo.js b/node_modules/kafkajs/src/utils/sharedPromiseTo.js new file mode 100644 index 0000000..48815ff --- /dev/null +++ b/node_modules/kafkajs/src/utils/sharedPromiseTo.js @@ -0,0 +1,18 @@ +/** + * @template T + * @param { (...args: any) => Promise } [asyncFunction] + * Promise returning function that will only ever be invoked sequentially. + * @returns { (...args: any) => Promise } + * Function that may invoke asyncFunction if there is not a currently executing invocation. + * Returns promise from the currently executing invocation. + */ +module.exports = asyncFunction => { + let promise = null + + return (...args) => { + if (promise == null) { + promise = asyncFunction(...args).finally(() => (promise = null)) + } + return promise + } +} diff --git a/node_modules/kafkajs/src/utils/shuffle.js b/node_modules/kafkajs/src/utils/shuffle.js new file mode 100644 index 0000000..85efaaf --- /dev/null +++ b/node_modules/kafkajs/src/utils/shuffle.js @@ -0,0 +1,25 @@ +/** + * @param {T[]} array + * @returns T[] + * @template T + */ +module.exports = array => { + if (!Array.isArray(array)) { + throw new TypeError("'array' is not an array") + } + + if (array.length < 2) { + return array + } + + const copy = array.slice() + + for (let i = copy.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)) + const temp = copy[i] + copy[i] = copy[j] + copy[j] = temp + } + + return copy +} diff --git a/node_modules/kafkajs/src/utils/sleep.js b/node_modules/kafkajs/src/utils/sleep.js new file mode 100644 index 0000000..a715e10 --- /dev/null +++ b/node_modules/kafkajs/src/utils/sleep.js @@ -0,0 +1,4 @@ +module.exports = timeInMs => + new Promise(resolve => { + setTimeout(resolve, timeInMs) + }) diff --git a/node_modules/kafkajs/src/utils/swapObject.js b/node_modules/kafkajs/src/utils/swapObject.js new file mode 100644 index 0000000..4b01aaf --- /dev/null +++ b/node_modules/kafkajs/src/utils/swapObject.js @@ -0,0 +1,3 @@ +const { keys } = Object +module.exports = object => + keys(object).reduce((result, key) => ({ ...result, [object[key]]: key }), {}) diff --git a/node_modules/kafkajs/src/utils/uniq.js b/node_modules/kafkajs/src/utils/uniq.js new file mode 100644 index 0000000..2f2c05a --- /dev/null +++ b/node_modules/kafkajs/src/utils/uniq.js @@ -0,0 +1,3 @@ +const uniq = arr => [...new Set(arr)] + +module.exports = uniq diff --git a/node_modules/kafkajs/src/utils/waitFor.js b/node_modules/kafkajs/src/utils/waitFor.js new file mode 100644 index 0000000..551b2d9 --- /dev/null +++ b/node_modules/kafkajs/src/utils/waitFor.js @@ -0,0 +1,50 @@ +const sleep = require('./sleep') +const { KafkaJSTimeout } = require('../errors') + +module.exports = ( + fn, + { delay = 50, maxWait = 10000, timeoutMessage = 'Timeout', ignoreTimeout = false } = {} +) => { + let timeoutId + let totalWait = 0 + let fulfilled = false + + const checkCondition = async (resolve, reject) => { + totalWait += delay + if (fulfilled) { + return + } + + await sleep(delay) + + try { + const result = await fn(totalWait) + if (result) { + fulfilled = true + clearTimeout(timeoutId) + return resolve(result) + } + + checkCondition(resolve, reject) + } catch (e) { + fulfilled = true + clearTimeout(timeoutId) + reject(e) + } + } + + return new Promise((resolve, reject) => { + checkCondition(resolve, reject) + + if (ignoreTimeout) { + return + } + + timeoutId = setTimeout(() => { + if (!fulfilled) { + fulfilled = true + return reject(new KafkaJSTimeout(timeoutMessage)) + } + }, maxWait) + }) +} diff --git a/node_modules/kafkajs/src/utils/websiteUrl.js b/node_modules/kafkajs/src/utils/websiteUrl.js new file mode 100644 index 0000000..cb37442 --- /dev/null +++ b/node_modules/kafkajs/src/utils/websiteUrl.js @@ -0,0 +1,7 @@ +const BASE_URL = 'https://kafka.js.org' +const stripLeading = char => str => (str.charAt(0) === char ? str.substring(1) : str) +const stripLeadingSlash = stripLeading('/') +const stripLeadingHash = stripLeading('#') + +module.exports = (path, hash) => + `${BASE_URL}/${stripLeadingSlash(path)}${hash ? '#' + stripLeadingHash(hash) : ''}` diff --git a/node_modules/kafkajs/types/index.d.ts b/node_modules/kafkajs/types/index.d.ts new file mode 100644 index 0000000..26bcbfe --- /dev/null +++ b/node_modules/kafkajs/types/index.d.ts @@ -0,0 +1,1311 @@ +/// + +import * as tls from 'tls' +import * as net from 'net' + +type Without = { [P in Exclude]?: never } +type XOR = T | U extends object ? (Without & U) | (Without & T) : T | U + +export class Kafka { + constructor(config: KafkaConfig) + producer(config?: ProducerConfig): Producer + consumer(config: ConsumerConfig): Consumer + admin(config?: AdminConfig): Admin + logger(): Logger +} + +export type BrokersFunction = () => string[] | Promise + +type SaslAuthenticationRequest = { + encode: () => Buffer | Promise +} +type SaslAuthenticationResponse = { + decode: (rawResponse: Buffer) => Buffer | Promise + parse: (data: Buffer) => ParseResult +} + +export type Authenticator = { + authenticate: () => Promise +} + +export type SaslAuthenticateArgs = { + request: SaslAuthenticationRequest + response?: SaslAuthenticationResponse +} + +export type AuthenticationProviderArgs = { + host: string + port: number + logger: Logger + saslAuthenticate: ( + args: SaslAuthenticateArgs + ) => Promise +} + +export type Mechanism = { + mechanism: string + authenticationProvider: (args: AuthenticationProviderArgs) => Authenticator +} + +export interface KafkaConfig { + brokers: string[] | BrokersFunction + ssl?: tls.ConnectionOptions | boolean + sasl?: SASLOptions | Mechanism + clientId?: string + connectionTimeout?: number + authenticationTimeout?: number + reauthenticationThreshold?: number + requestTimeout?: number + enforceRequestTimeout?: boolean + retry?: RetryOptions + socketFactory?: ISocketFactory + logLevel?: logLevel + logCreator?: logCreator +} + +export interface ISocketFactoryArgs { + host: string + port: number + ssl: tls.ConnectionOptions + onConnect: () => void +} + +export type ISocketFactory = (args: ISocketFactoryArgs) => net.Socket + +export interface OauthbearerProviderResponse { + value: string +} + +type SASLMechanismOptionsMap = { + plain: { username: string; password: string } + 'scram-sha-256': { username: string; password: string } + 'scram-sha-512': { username: string; password: string } + aws: { + authorizationIdentity: string + accessKeyId: string + secretAccessKey: string + sessionToken?: string + } + oauthbearer: { oauthBearerProvider: () => Promise } +} + +export type SASLMechanism = keyof SASLMechanismOptionsMap +type SASLMechanismOptions = T extends SASLMechanism + ? { mechanism: T } & SASLMechanismOptionsMap[T] + : never +export type SASLOptions = SASLMechanismOptions + +export interface ProducerConfig { + createPartitioner?: ICustomPartitioner + retry?: RetryOptions + metadataMaxAge?: number + allowAutoTopicCreation?: boolean + idempotent?: boolean + transactionalId?: string + transactionTimeout?: number + maxInFlightRequests?: number +} + +export interface Message { + key?: Buffer | string | null + value: Buffer | string | null + partition?: number + headers?: IHeaders + timestamp?: string +} + +export interface PartitionerArgs { + topic: string + partitionMetadata: PartitionMetadata[] + message: Message +} + +export type ICustomPartitioner = () => (args: PartitionerArgs) => number +export type DefaultPartitioner = ICustomPartitioner +export type LegacyPartitioner = ICustomPartitioner + +export const Partitioners: { + DefaultPartitioner: DefaultPartitioner + LegacyPartitioner: LegacyPartitioner + /** + * @deprecated Use DefaultPartitioner instead + * + * The JavaCompatiblePartitioner was renamed DefaultPartitioner + * and made to be the default in 2.0.0. + */ + JavaCompatiblePartitioner: DefaultPartitioner +} + +export type PartitionMetadata = { + partitionErrorCode: number + partitionId: number + leader: number + replicas: number[] + isr: number[] + offlineReplicas?: number[] +} + +export interface IHeaders { + [key: string]: Buffer | string | (Buffer | string)[] | undefined +} + +export interface ConsumerConfig { + groupId: string + partitionAssigners?: PartitionAssigner[] + metadataMaxAge?: number + sessionTimeout?: number + rebalanceTimeout?: number + heartbeatInterval?: number + maxBytesPerPartition?: number + minBytes?: number + maxBytes?: number + maxWaitTimeInMs?: number + retry?: RetryOptions & { restartOnFailure?: (err: Error) => Promise } + allowAutoTopicCreation?: boolean + maxInFlightRequests?: number + readUncommitted?: boolean + rackId?: string +} + +export type PartitionAssigner = (config: { + cluster: Cluster + groupId: string + logger: Logger +}) => Assigner + +export interface CoordinatorMetadata { + errorCode: number + coordinator: { + nodeId: number + host: string + port: number + } +} + +export type Cluster = { + getNodeIds(): number[] + metadata(): Promise + removeBroker(options: { host: string; port: number }): void + addMultipleTargetTopics(topics: string[]): Promise + isConnected(): boolean + connect(): Promise + disconnect(): Promise + refreshMetadata(): Promise + refreshMetadataIfNecessary(): Promise + addTargetTopic(topic: string): Promise + findBroker(node: { nodeId: string }): Promise + findControllerBroker(): Promise + findTopicPartitionMetadata(topic: string): PartitionMetadata[] + findLeaderForPartitions(topic: string, partitions: number[]): { [leader: string]: number[] } + findGroupCoordinator(group: { groupId: string }): Promise + findGroupCoordinatorMetadata(group: { groupId: string }): Promise + defaultOffset(config: { fromBeginning: boolean }): number + fetchTopicsOffset( + topics: Array< + { + topic: string + partitions: Array<{ partition: number }> + } & XOR<{ fromBeginning: boolean }, { fromTimestamp: number }> + > + ): Promise +} + +export type Assignment = { [topic: string]: number[] } + +export type GroupMember = { memberId: string; memberMetadata: Buffer } + +export type GroupMemberAssignment = { memberId: string; memberAssignment: Buffer } + +export type GroupState = { name: string; metadata: Buffer } + +export type Assigner = { + name: string + version: number + assign(group: { members: GroupMember[]; topics: string[] }): Promise + protocol(subscription: { topics: string[] }): GroupState +} + +export interface RetryOptions { + maxRetryTime?: number + initialRetryTime?: number + factor?: number + multiplier?: number + retries?: number + restartOnFailure?: (e: Error) => Promise +} + +export interface AdminConfig { + retry?: RetryOptions +} + +export interface ITopicConfig { + topic: string + numPartitions?: number + replicationFactor?: number + replicaAssignment?: ReplicaAssignment[] + configEntries?: IResourceConfigEntry[] +} + +export interface ITopicPartitionConfig { + topic: string + count: number + assignments?: Array> +} + +export interface ITopicMetadata { + name: string + partitions: PartitionMetadata[] +} + +export interface ReplicaAssignment { + partition: number + replicas: Array +} + +export interface PartitionReassignment { + topic: string + partitionAssignment: Array +} + +export enum AclResourceTypes { + UNKNOWN = 0, + ANY = 1, + TOPIC = 2, + GROUP = 3, + CLUSTER = 4, + TRANSACTIONAL_ID = 5, + DELEGATION_TOKEN = 6, +} + +export enum ConfigResourceTypes { + UNKNOWN = 0, + TOPIC = 2, + BROKER = 4, + BROKER_LOGGER = 8, +} + +export enum ConfigSource { + UNKNOWN = 0, + TOPIC_CONFIG = 1, + DYNAMIC_BROKER_CONFIG = 2, + DYNAMIC_DEFAULT_BROKER_CONFIG = 3, + STATIC_BROKER_CONFIG = 4, + DEFAULT_CONFIG = 5, + DYNAMIC_BROKER_LOGGER_CONFIG = 6, +} + +export enum AclPermissionTypes { + UNKNOWN = 0, + ANY = 1, + DENY = 2, + ALLOW = 3, +} + +export enum AclOperationTypes { + UNKNOWN = 0, + ANY = 1, + ALL = 2, + READ = 3, + WRITE = 4, + CREATE = 5, + DELETE = 6, + ALTER = 7, + DESCRIBE = 8, + CLUSTER_ACTION = 9, + DESCRIBE_CONFIGS = 10, + ALTER_CONFIGS = 11, + IDEMPOTENT_WRITE = 12, +} + +export enum ResourcePatternTypes { + UNKNOWN = 0, + ANY = 1, + MATCH = 2, + LITERAL = 3, + PREFIXED = 4, +} + +export interface ResourceConfigQuery { + type: ConfigResourceTypes + name: string + configNames?: string[] +} + +export interface ConfigEntries { + configName: string + configValue: string + isDefault: boolean + configSource: ConfigSource + isSensitive: boolean + readOnly: boolean + configSynonyms: ConfigSynonyms[] +} + +export interface ConfigSynonyms { + configName: string + configValue: string + configSource: ConfigSource +} + +export interface DescribeConfigResponse { + resources: { + configEntries: ConfigEntries[] + errorCode: number + errorMessage: string + resourceName: string + resourceType: ConfigResourceTypes + }[] + throttleTime: number +} + +export interface IResourceConfigEntry { + name: string + value: string +} + +export interface IResourceConfig { + type: ConfigResourceTypes + name: string + configEntries: IResourceConfigEntry[] +} + +type ValueOf = T[keyof T] + +export type AdminEvents = { + CONNECT: 'admin.connect' + DISCONNECT: 'admin.disconnect' + REQUEST: 'admin.network.request' + REQUEST_TIMEOUT: 'admin.network.request_timeout' + REQUEST_QUEUE_SIZE: 'admin.network.request_queue_size' +} + +export interface InstrumentationEvent { + id: string + type: string + timestamp: number + payload: T +} + +export type RemoveInstrumentationEventListener = () => void + +export type ConnectEvent = InstrumentationEvent +export type DisconnectEvent = InstrumentationEvent +export type RequestEvent = InstrumentationEvent<{ + apiKey: number + apiName: string + apiVersion: number + broker: string + clientId: string + correlationId: number + createdAt: number + duration: number + pendingDuration: number + sentAt: number + size: number +}> +export type RequestTimeoutEvent = InstrumentationEvent<{ + apiKey: number + apiName: string + apiVersion: number + broker: string + clientId: string + correlationId: number + createdAt: number + pendingDuration: number + sentAt: number +}> +export type RequestQueueSizeEvent = InstrumentationEvent<{ + broker: string + clientId: string + queueSize: number +}> + +export type SeekEntry = PartitionOffset + +export type FetchOffsetsPartition = PartitionOffset & { metadata: string | null } +export interface Acl { + principal: string + host: string + operation: AclOperationTypes + permissionType: AclPermissionTypes +} + +export interface AclResource { + resourceType: AclResourceTypes + resourceName: string + resourcePatternType: ResourcePatternTypes +} + +export type AclEntry = Acl & AclResource + +export type DescribeAclResource = AclResource & { + acls: Acl[] +} + +export interface DescribeAclResponse { + throttleTime: number + errorCode: number + errorMessage?: string + resources: DescribeAclResource[] +} + +export interface AclFilter { + resourceType: AclResourceTypes + resourceName?: string + resourcePatternType: ResourcePatternTypes + principal?: string + host?: string + operation: AclOperationTypes + permissionType: AclPermissionTypes +} + +export interface MatchingAcl { + errorCode: number + errorMessage?: string + resourceType: AclResourceTypes + resourceName: string + resourcePatternType: ResourcePatternTypes + principal: string + host: string + operation: AclOperationTypes + permissionType: AclPermissionTypes +} + +export interface DeleteAclFilterResponses { + errorCode: number + errorMessage?: string + matchingAcls: MatchingAcl[] +} + +export interface DeleteAclResponse { + throttleTime: number + filterResponses: DeleteAclFilterResponses[] +} + +export interface ListPartitionReassignmentsResponse { + topics: OngoingTopicReassignment[] +} + +export interface OngoingTopicReassignment { + topic: string + partitions: OngoingPartitionReassignment[] +} + +export interface OngoingPartitionReassignment { + partitionIndex: number + replicas: number[] + addingReplicas?: number[] + removingReplicas?: number[] +} + +export type Admin = { + connect(): Promise + disconnect(): Promise + listTopics(): Promise + createTopics(options: { + validateOnly?: boolean + waitForLeaders?: boolean + timeout?: number + topics: ITopicConfig[] + }): Promise + deleteTopics(options: { topics: string[]; timeout?: number }): Promise + createPartitions(options: { + validateOnly?: boolean + timeout?: number + topicPartitions: ITopicPartitionConfig[] + }): Promise + fetchTopicMetadata(options?: { topics: string[] }): Promise<{ topics: Array }> + fetchOffsets(options: { + groupId: string + topics?: string[] + resolveOffsets?: boolean + }): Promise> + fetchTopicOffsets(topic: string): Promise> + fetchTopicOffsetsByTimestamp(topic: string, timestamp?: number): Promise> + describeCluster(): Promise<{ + brokers: Array<{ nodeId: number; host: string; port: number }> + controller: number | null + clusterId: string + }> + setOffsets(options: { groupId: string; topic: string; partitions: SeekEntry[] }): Promise + resetOffsets(options: { groupId: string; topic: string; earliest: boolean }): Promise + describeConfigs(configs: { + resources: ResourceConfigQuery[] + includeSynonyms: boolean + }): Promise + alterConfigs(configs: { validateOnly: boolean; resources: IResourceConfig[] }): Promise + listGroups(): Promise<{ groups: GroupOverview[] }> + deleteGroups(groupIds: string[]): Promise + describeGroups(groupIds: string[]): Promise + describeAcls(options: AclFilter): Promise + deleteAcls(options: { filters: AclFilter[] }): Promise + createAcls(options: { acl: AclEntry[] }): Promise + deleteTopicRecords(options: { topic: string; partitions: SeekEntry[] }): Promise + alterPartitionReassignments(request: { + topics: PartitionReassignment[] + timeout?: number + }): Promise + listPartitionReassignments(request: { + topics?: TopicPartitions[] + timeout?: number + }): Promise + logger(): Logger + on( + eventName: AdminEvents['CONNECT'], + listener: (event: ConnectEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: AdminEvents['DISCONNECT'], + listener: (event: DisconnectEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: AdminEvents['REQUEST'], + listener: (event: RequestEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: AdminEvents['REQUEST_QUEUE_SIZE'], + listener: (event: RequestQueueSizeEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: AdminEvents['REQUEST_TIMEOUT'], + listener: (event: RequestTimeoutEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ValueOf, + listener: (event: InstrumentationEvent) => void + ): RemoveInstrumentationEventListener + readonly events: AdminEvents +} + +export const PartitionAssigners: { roundRobin: PartitionAssigner } + +export interface ISerializer { + encode(value: T): Buffer + decode(buffer: Buffer): T | null +} + +export type MemberMetadata = { + version: number + topics: string[] + userData: Buffer +} + +export type MemberAssignment = { + version: number + assignment: Assignment + userData: Buffer +} + +export const AssignerProtocol: { + MemberMetadata: ISerializer + MemberAssignment: ISerializer +} + +export enum logLevel { + NOTHING = 0, + ERROR = 1, + WARN = 2, + INFO = 4, + DEBUG = 5, +} + +export interface LogEntry { + namespace: string + level: logLevel + label: string + log: LoggerEntryContent +} + +export interface LoggerEntryContent { + readonly timestamp: string + readonly message: string + [key: string]: any +} + +export type logCreator = (logLevel: logLevel) => (entry: LogEntry) => void + +export type Logger = { + info: (message: string, extra?: object) => void + error: (message: string, extra?: object) => void + warn: (message: string, extra?: object) => void + debug: (message: string, extra?: object) => void + + namespace: (namespace: string, logLevel?: logLevel) => Logger + setLogLevel: (logLevel: logLevel) => void +} + +export interface BrokerMetadata { + brokers: Array<{ nodeId: number; host: string; port: number; rack?: string }> + topicMetadata: Array<{ + topicErrorCode: number + topic: string + partitionMetadata: PartitionMetadata[] + }> +} + +export interface ApiVersions { + [apiKey: number]: { + minVersion: number + maxVersion: number + } +} + +export type Broker = { + isConnected(): boolean + connect(): Promise + disconnect(): Promise + apiVersions(): Promise + metadata(topics: string[]): Promise + describeGroups: (options: { groupIds: string[] }) => Promise + offsetCommit(request: { + groupId: string + groupGenerationId: number + memberId: string + retentionTime?: number + topics: TopicOffsets[] + }): Promise + offsetFetch(request: { + groupId: string + topics: TopicOffsets[] + }): Promise<{ + responses: TopicOffsets[] + }> + fetch(request: { + replicaId?: number + isolationLevel?: number + maxWaitTime?: number + minBytes?: number + maxBytes?: number + topics: Array<{ + topic: string + partitions: Array<{ partition: number; fetchOffset: string; maxBytes: number }> + }> + rackId?: string + }): Promise + produce(request: { + topicData: Array<{ + topic: string + partitions: Array<{ partition: number; firstSequence?: number; messages: Message[] }> + }> + transactionalId?: string + producerId?: number + producerEpoch?: number + acks?: number + timeout?: number + compression?: CompressionTypes + }): Promise + alterPartitionReassignments(request: { + topics: PartitionReassignment[] + timeout?: number + }): Promise + listPartitionReassignments(request: { + topics?: TopicPartitions[] + timeout?: number + }): Promise +} + +interface MessageSetEntry { + key: Buffer | null + value: Buffer | null + timestamp: string + attributes: number + offset: string + size: number + headers?: never +} + +interface RecordBatchEntry { + key: Buffer | null + value: Buffer | null + timestamp: string + attributes: number + offset: string + headers: IHeaders + size?: never +} + +export type KafkaMessage = MessageSetEntry | RecordBatchEntry + +export interface ProducerRecord { + topic: string + messages: Message[] + acks?: number + timeout?: number + compression?: CompressionTypes +} + +export type RecordMetadata = { + topicName: string + partition: number + errorCode: number + offset?: string + timestamp?: string + baseOffset?: string + logAppendTime?: string + logStartOffset?: string +} + +export interface TopicMessages { + topic: string + messages: Message[] +} + +export interface ProducerBatch { + acks?: number + timeout?: number + compression?: CompressionTypes + topicMessages?: TopicMessages[] +} + +export interface PartitionOffset { + partition: number + offset: string +} + +export interface TopicOffsets { + topic: string + partitions: PartitionOffset[] +} + +export interface Offsets { + topics: TopicOffsets[] +} + +type Sender = { + send(record: ProducerRecord): Promise + sendBatch(batch: ProducerBatch): Promise +} + +export type ProducerEvents = { + CONNECT: 'producer.connect' + DISCONNECT: 'producer.disconnect' + REQUEST: 'producer.network.request' + REQUEST_TIMEOUT: 'producer.network.request_timeout' + REQUEST_QUEUE_SIZE: 'producer.network.request_queue_size' +} + +export type Producer = Sender & { + connect(): Promise + disconnect(): Promise + isIdempotent(): boolean + readonly events: ProducerEvents + on( + eventName: ProducerEvents['CONNECT'], + listener: (event: ConnectEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ProducerEvents['DISCONNECT'], + listener: (event: DisconnectEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ProducerEvents['REQUEST'], + listener: (event: RequestEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ProducerEvents['REQUEST_QUEUE_SIZE'], + listener: (event: RequestQueueSizeEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ProducerEvents['REQUEST_TIMEOUT'], + listener: (event: RequestTimeoutEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ValueOf, + listener: (event: InstrumentationEvent) => void + ): RemoveInstrumentationEventListener + transaction(): Promise + logger(): Logger +} + +export type Transaction = Sender & { + sendOffsets(offsets: Offsets & { consumerGroupId: string }): Promise + commit(): Promise + abort(): Promise + isActive(): boolean +} + +export type ConsumerGroup = { + groupId: string + generationId: number + memberId: string + coordinator: Broker +} + +export type MemberDescription = { + clientHost: string + clientId: string + memberId: string + memberAssignment: Buffer + memberMetadata: Buffer +} + +// See https://github.com/apache/kafka/blob/2.4.0/clients/src/main/java/org/apache/kafka/common/ConsumerGroupState.java#L25 +export type ConsumerGroupState = + | 'Unknown' + | 'PreparingRebalance' + | 'CompletingRebalance' + | 'Stable' + | 'Dead' + | 'Empty' + +export type GroupDescription = { + groupId: string + members: MemberDescription[] + protocol: string + protocolType: string + state: ConsumerGroupState +} + +export type GroupDescriptions = { + groups: GroupDescription[] +} + +export type TopicPartitions = { topic: string; partitions: number[] } + +export type TopicPartition = { + topic: string + partition: number +} +export type TopicPartitionOffset = TopicPartition & { + offset: string +} +export type TopicPartitionOffsetAndMetadata = TopicPartitionOffset & { + metadata?: string | null +} + +export type Batch = { + topic: string + partition: number + highWatermark: string + messages: KafkaMessage[] + isEmpty(): boolean + firstOffset(): string | null + lastOffset(): string + offsetLag(): string + offsetLagLow(): string +} + +export type GroupOverview = { + groupId: string + protocolType: string +} + +export type DeleteGroupsResult = { + groupId: string + errorCode?: number + error?: KafkaJSProtocolError +} + +export type ConsumerEvents = { + HEARTBEAT: 'consumer.heartbeat' + COMMIT_OFFSETS: 'consumer.commit_offsets' + GROUP_JOIN: 'consumer.group_join' + FETCH_START: 'consumer.fetch_start' + FETCH: 'consumer.fetch' + START_BATCH_PROCESS: 'consumer.start_batch_process' + END_BATCH_PROCESS: 'consumer.end_batch_process' + CONNECT: 'consumer.connect' + DISCONNECT: 'consumer.disconnect' + STOP: 'consumer.stop' + CRASH: 'consumer.crash' + REBALANCING: 'consumer.rebalancing' + RECEIVED_UNSUBSCRIBED_TOPICS: 'consumer.received_unsubscribed_topics' + REQUEST: 'consumer.network.request' + REQUEST_TIMEOUT: 'consumer.network.request_timeout' + REQUEST_QUEUE_SIZE: 'consumer.network.request_queue_size' +} +export type ConsumerHeartbeatEvent = InstrumentationEvent<{ + groupId: string + memberId: string + groupGenerationId: number +}> +export type ConsumerCommitOffsetsEvent = InstrumentationEvent<{ + groupId: string + memberId: string + groupGenerationId: number + topics: TopicOffsets[] +}> +export interface IMemberAssignment { + [key: string]: number[] +} +export type ConsumerGroupJoinEvent = InstrumentationEvent<{ + duration: number + groupId: string + isLeader: boolean + leaderId: string + groupProtocol: string + memberId: string + memberAssignment: IMemberAssignment +}> +export type ConsumerFetchStartEvent = InstrumentationEvent<{ nodeId: number }> +export type ConsumerFetchEvent = InstrumentationEvent<{ + numberOfBatches: number + duration: number + nodeId: number +}> +interface IBatchProcessEvent { + topic: string + partition: number + highWatermark: string + offsetLag: string + offsetLagLow: string + batchSize: number + firstOffset: string + lastOffset: string +} +export type ConsumerStartBatchProcessEvent = InstrumentationEvent +export type ConsumerEndBatchProcessEvent = InstrumentationEvent< + IBatchProcessEvent & { duration: number } +> +export type ConsumerCrashEvent = InstrumentationEvent<{ + error: Error + groupId: string + restart: boolean +}> +export type ConsumerRebalancingEvent = InstrumentationEvent<{ + groupId: string + memberId: string +}> +export type ConsumerReceivedUnsubcribedTopicsEvent = InstrumentationEvent<{ + groupId: string + generationId: number + memberId: string + assignedTopics: string[] + topicsSubscribed: string[] + topicsNotSubscribed: string[] +}> + +export interface OffsetsByTopicPartition { + topics: TopicOffsets[] +} + +export interface EachMessagePayload { + topic: string + partition: number + message: KafkaMessage + heartbeat(): Promise + pause(): () => void +} + +export interface EachBatchPayload { + batch: Batch + resolveOffset(offset: string): void + heartbeat(): Promise + pause(): () => void + commitOffsetsIfNecessary(offsets?: Offsets): Promise + uncommittedOffsets(): OffsetsByTopicPartition + isRunning(): boolean + isStale(): boolean +} + +/** + * Type alias to keep compatibility with @types/kafkajs + * @see https://github.com/DefinitelyTyped/DefinitelyTyped/blob/712ad9d59ccca6a3cc92f347fea0d1c7b02f5eeb/types/kafkajs/index.d.ts#L321-L325 + */ +export type ConsumerEachMessagePayload = EachMessagePayload + +/** + * Type alias to keep compatibility with @types/kafkajs + * @see https://github.com/DefinitelyTyped/DefinitelyTyped/blob/712ad9d59ccca6a3cc92f347fea0d1c7b02f5eeb/types/kafkajs/index.d.ts#L327-L336 + */ +export type ConsumerEachBatchPayload = EachBatchPayload + +export type EachBatchHandler = (payload: EachBatchPayload) => Promise +export type EachMessageHandler = (payload: EachMessagePayload) => Promise + +export type ConsumerRunConfig = { + autoCommit?: boolean + autoCommitInterval?: number | null + autoCommitThreshold?: number | null + eachBatchAutoResolve?: boolean + partitionsConsumedConcurrently?: number + eachBatch?: EachBatchHandler + eachMessage?: EachMessageHandler +} + +/** + * @deprecated Replaced by ConsumerSubscribeTopics + */ +export type ConsumerSubscribeTopic = { topic: string | RegExp; fromBeginning?: boolean } +export type ConsumerSubscribeTopics = { topics: (string | RegExp)[]; fromBeginning?: boolean } + +export type Consumer = { + connect(): Promise + disconnect(): Promise + subscribe(subscription: ConsumerSubscribeTopics | ConsumerSubscribeTopic): Promise + stop(): Promise + run(config?: ConsumerRunConfig): Promise + commitOffsets(topicPartitions: Array): Promise + seek(topicPartitionOffset: TopicPartitionOffset): void + describeGroup(): Promise + pause(topics: Array<{ topic: string; partitions?: number[] }>): void + paused(): TopicPartitions[] + resume(topics: Array<{ topic: string; partitions?: number[] }>): void + on( + eventName: ConsumerEvents['HEARTBEAT'], + listener: (event: ConsumerHeartbeatEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['COMMIT_OFFSETS'], + listener: (event: ConsumerCommitOffsetsEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['GROUP_JOIN'], + listener: (event: ConsumerGroupJoinEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['FETCH_START'], + listener: (event: ConsumerFetchStartEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['FETCH'], + listener: (event: ConsumerFetchEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['START_BATCH_PROCESS'], + listener: (event: ConsumerStartBatchProcessEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['END_BATCH_PROCESS'], + listener: (event: ConsumerEndBatchProcessEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['CONNECT'], + listener: (event: ConnectEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['DISCONNECT'], + listener: (event: DisconnectEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['STOP'], + listener: (event: InstrumentationEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['CRASH'], + listener: (event: ConsumerCrashEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['REBALANCING'], + listener: (event: ConsumerRebalancingEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['RECEIVED_UNSUBSCRIBED_TOPICS'], + listener: (event: ConsumerReceivedUnsubcribedTopicsEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['REQUEST'], + listener: (event: RequestEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['REQUEST_TIMEOUT'], + listener: (event: RequestTimeoutEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ConsumerEvents['REQUEST_QUEUE_SIZE'], + listener: (event: RequestQueueSizeEvent) => void + ): RemoveInstrumentationEventListener + on( + eventName: ValueOf, + listener: (event: InstrumentationEvent) => void + ): RemoveInstrumentationEventListener + logger(): Logger + readonly events: ConsumerEvents +} + +export enum CompressionTypes { + None = 0, + GZIP = 1, + Snappy = 2, + LZ4 = 3, + ZSTD = 4, +} + +export var CompressionCodecs: { + [CompressionTypes.GZIP]: () => any + [CompressionTypes.Snappy]: () => any + [CompressionTypes.LZ4]: () => any + [CompressionTypes.ZSTD]: () => any +} + +export class KafkaJSError extends Error { + readonly message: Error['message'] + readonly name: string + readonly retriable: boolean + readonly helpUrl?: string + readonly cause?: Error + + constructor(e: Error | string, metadata?: KafkaJSErrorMetadata) +} + +export class KafkaJSNonRetriableError extends KafkaJSError { + constructor(e: Error | string) +} + +export class KafkaJSProtocolError extends KafkaJSError { + readonly code: number + readonly type: string + constructor(e: Error | string) +} + +export class KafkaJSAggregateError extends Error { + readonly errors: (Error | string)[] + constructor(message: Error | string, errors: (Error | string)[]) +} + +export class KafkaJSOffsetOutOfRange extends KafkaJSProtocolError { + readonly topic: string + readonly partition: number + constructor(e: Error | string, metadata?: KafkaJSOffsetOutOfRangeMetadata) +} + +export class KafkaJSAlterPartitionReassignmentsError extends KafkaJSProtocolError { + readonly topic?: string + readonly partition?: number + constructor(e: Error | string, topic?: string, partition?: number) +} + +export class KafkaJSNumberOfRetriesExceeded extends KafkaJSNonRetriableError { + readonly stack: string + readonly retryCount: number + readonly retryTime: number + constructor(e: Error | string, metadata?: KafkaJSNumberOfRetriesExceededMetadata) +} + +export class KafkaJSConnectionError extends KafkaJSError { + readonly broker: string + constructor(e: Error | string, metadata?: KafkaJSConnectionErrorMetadata) +} + +export class KafkaJSRequestTimeoutError extends KafkaJSError { + readonly broker: string + readonly correlationId: number + readonly createdAt: number + readonly sentAt: number + readonly pendingDuration: number + constructor(e: Error | string, metadata?: KafkaJSRequestTimeoutErrorMetadata) +} + +export class KafkaJSMetadataNotLoaded extends KafkaJSError { + constructor() +} + +export class KafkaJSTopicMetadataNotLoaded extends KafkaJSMetadataNotLoaded { + readonly topic: string + constructor(e: Error | string, metadata?: KafkaJSTopicMetadataNotLoadedMetadata) +} + +export class KafkaJSStaleTopicMetadataAssignment extends KafkaJSError { + readonly topic: string + readonly unknownPartitions: number + constructor(e: Error | string, metadata?: KafkaJSStaleTopicMetadataAssignmentMetadata) +} + +export class KafkaJSServerDoesNotSupportApiKey extends KafkaJSNonRetriableError { + readonly apiKey: number + readonly apiName: string + constructor(e: Error | string, metadata?: KafkaJSServerDoesNotSupportApiKeyMetadata) +} + +export class KafkaJSBrokerNotFound extends KafkaJSError { + constructor() +} + +export class KafkaJSPartialMessageError extends KafkaJSError { + constructor() +} + +export class KafkaJSSASLAuthenticationError extends KafkaJSError { + constructor() +} + +export class KafkaJSGroupCoordinatorNotFound extends KafkaJSError { + constructor() +} + +export class KafkaJSNotImplemented extends KafkaJSError { + constructor() +} + +export class KafkaJSTimeout extends KafkaJSError { + constructor() +} + +export class KafkaJSLockTimeout extends KafkaJSError { + constructor() +} + +export class KafkaJSUnsupportedMagicByteInMessageSet extends KafkaJSError { + constructor() +} + +export class KafkaJSDeleteGroupsError extends KafkaJSError { + readonly groups: DeleteGroupsResult[] + constructor(e: Error | string, groups?: KafkaJSDeleteGroupsErrorGroups[]) +} + +export class KafkaJSDeleteTopicRecordsError extends KafkaJSError { + constructor(metadata: KafkaJSDeleteTopicRecordsErrorTopic) +} + +export interface KafkaJSDeleteGroupsErrorGroups { + groupId: string + errorCode: number + error: KafkaJSError +} + +export interface KafkaJSDeleteTopicRecordsErrorTopic { + topic: string + partitions: KafkaJSDeleteTopicRecordsErrorPartition[] +} + +export interface KafkaJSDeleteTopicRecordsErrorPartition { + partition: number + offset: string + error: KafkaJSError +} + +export interface KafkaJSErrorMetadata { + retriable?: boolean + topic?: string + partitionId?: number + metadata?: PartitionMetadata +} + +export interface KafkaJSOffsetOutOfRangeMetadata { + topic: string + partition: number +} + +export interface KafkaJSNumberOfRetriesExceededMetadata { + retryCount: number + retryTime: number +} + +export interface KafkaJSConnectionErrorMetadata { + broker?: string + code?: string +} + +export interface KafkaJSRequestTimeoutErrorMetadata { + broker: string + clientId: string + correlationId: number + createdAt: number + sentAt: number + pendingDuration: number +} + +export interface KafkaJSTopicMetadataNotLoadedMetadata { + topic: string +} + +export interface KafkaJSStaleTopicMetadataAssignmentMetadata { + topic: string + unknownPartitions: PartitionMetadata[] +} + +export interface KafkaJSServerDoesNotSupportApiKeyMetadata { + apiKey: number + apiName: string +} diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..1f17217 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,24 @@ +{ + "name": "near-intents-monitor-poc", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "near-intents-monitor-poc", + "version": "0.1.0", + "dependencies": { + "kafkajs": "^2.2.4" + } + }, + "node_modules/kafkajs": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/kafkajs/-/kafkajs-2.2.4.tgz", + "integrity": "sha512-j/YeapB1vfPT2iOIUn/vxdyKEuhuY2PxMBvf5JWux6iSaukAccrMtXEY/Lb7OvavDhOWME589bpLrEdnVHjfjA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..cf29be7 --- /dev/null +++ b/package.json @@ -0,0 +1,14 @@ +{ + "name": "near-intents-monitor-poc", + "version": "0.1.0", + "private": true, + "type": "module", + "scripts": { + "near-intents:ingest": "node src/apps/near-intents-ingest.mjs", + "dummy-consumer": "node src/apps/dummy-consumer.mjs", + "start": "node index.mjs" + }, + "dependencies": { + "kafkajs": "^2.2.4" + } +} diff --git a/src/apps/dummy-consumer.mjs b/src/apps/dummy-consumer.mjs new file mode 100644 index 0000000..82a0aea --- /dev/null +++ b/src/apps/dummy-consumer.mjs @@ -0,0 +1,46 @@ +import process from 'node:process'; + +import { createConsumer } from '../bus/kafka/consumer.mjs'; +import { logStatus } from '../core/log.mjs'; +import { loadConfig } from '../lib/config.mjs'; + +const config = loadConfig(); + +const consumer = await createConsumer({ + groupId: config.kafkaConsumerGroupDummy, + brokers: config.kafkaBrokers, + clientId: config.kafkaClientId, +}); +await consumer.subscribe({ topic: config.kafkaTopicNormSwapDemand, fromBeginning: false }); +logStatus( + `dummy consumer subscribed to ${config.kafkaTopicNormSwapDemand} as ${config.kafkaConsumerGroupDummy}`, +); + +process.on('SIGINT', async () => { + await consumer.disconnect(); + process.exit(0); +}); + +process.on('SIGTERM', async () => { + await consumer.disconnect(); + process.exit(0); +}); + +await consumer.run({ + eachMessage: async ({ message }) => { + if (!message.value) return; + + let event; + try { + event = JSON.parse(message.value.toString()); + } catch { + logStatus('dummy consumer received non-JSON message; skipping'); + return; + } + + const payload = event?.payload || {}; + const pair = `${payload.assetIn || '?'} -> ${payload.assetOut || '?'}`; + const quoteId = payload.quoteId || event.eventId || '?'; + console.log(`[dummy-reactor] observed ${pair} quote_id=${quoteId} | would decide later`); + }, +}); diff --git a/src/apps/near-intents-ingest.mjs b/src/apps/near-intents-ingest.mjs new file mode 100644 index 0000000..edd9c8a --- /dev/null +++ b/src/apps/near-intents-ingest.mjs @@ -0,0 +1,40 @@ +import process from 'node:process'; + +import { createProducer } from '../bus/kafka/producer.mjs'; +import { logStatus } from '../core/log.mjs'; +import { parsePairFilter } from '../core/pair-filter.mjs'; +import { loadConfig } from '../lib/config.mjs'; +import { startNearIntentsWs } from '../venues/near-intents/ws.mjs'; + +const config = loadConfig(); +const pairFilter = parsePairFilter(process.argv.slice(2)); + +if (!config.nearIntentsApiKey) { + console.error('Missing NEAR_INTENTS_API_KEY in env or .env'); + process.exit(1); +} + +const producer = await createProducer({ + brokers: config.kafkaBrokers, + clientId: config.kafkaClientId, +}); +logStatus(`kafka producer connected; topic=${config.kafkaTopicNormSwapDemand}`); +if (pairFilter) logStatus(`pair filter enabled: ${pairFilter[0]} <-> ${pairFilter[1]}`); + +process.on('SIGINT', async () => { + await producer.disconnect(); + process.exit(0); +}); + +process.on('SIGTERM', async () => { + await producer.disconnect(); + process.exit(0); +}); + +await startNearIntentsWs({ + apiKey: config.nearIntentsApiKey, + wsUrl: config.nearIntentsWsUrl, + pairFilter, + producer, + topic: config.kafkaTopicNormSwapDemand, +}); diff --git a/src/bus/kafka.mjs b/src/bus/kafka.mjs new file mode 100644 index 0000000..091dc27 --- /dev/null +++ b/src/bus/kafka.mjs @@ -0,0 +1,27 @@ +import { Kafka } from 'kafkajs'; + +function brokersFromEnv() { + return (process.env.KAFKA_BROKERS || '127.0.0.1:9092') + .split(',') + .map((x) => x.trim()) + .filter(Boolean); +} + +export function createKafka() { + return new Kafka({ + clientId: process.env.KAFKA_CLIENT_ID || 'trading-system', + brokers: brokersFromEnv(), + }); +} + +export async function createProducer() { + const producer = createKafka().producer(); + await producer.connect(); + return producer; +} + +export async function createConsumer({ groupId }) { + const consumer = createKafka().consumer({ groupId }); + await consumer.connect(); + return consumer; +} diff --git a/src/bus/kafka/consumer.mjs b/src/bus/kafka/consumer.mjs new file mode 100644 index 0000000..ceb886c --- /dev/null +++ b/src/bus/kafka/consumer.mjs @@ -0,0 +1,16 @@ +import { Kafka } from 'kafkajs'; + +function createKafka({ brokers = ['127.0.0.1:9092'], clientId = 'trading-system' } = {}) { + return new Kafka({ clientId, brokers }); +} + +export async function createConsumer({ groupId, ...options }) { + const consumer = createKafka(options).consumer({ groupId }); + await consumer.connect(); + + return { + subscribe: (options) => consumer.subscribe(options), + run: (options) => consumer.run(options), + disconnect: () => consumer.disconnect(), + }; +} diff --git a/src/bus/kafka/producer.mjs b/src/bus/kafka/producer.mjs new file mode 100644 index 0000000..f674440 --- /dev/null +++ b/src/bus/kafka/producer.mjs @@ -0,0 +1,21 @@ +import { Kafka } from 'kafkajs'; + +function createKafka({ brokers = ['127.0.0.1:9092'], clientId = 'trading-system' } = {}) { + return new Kafka({ clientId, brokers }); +} + +export async function createProducer(options = {}) { + const producer = createKafka(options).producer(); + await producer.connect(); + return { + async sendJson(topic, event, { key = event?.eventId ?? event?.key ?? null } = {}) { + await producer.send({ + topic, + messages: [{ key, value: JSON.stringify(event) }], + }); + }, + async disconnect() { + await producer.disconnect(); + }, + }; +} diff --git a/src/core/env.mjs b/src/core/env.mjs new file mode 100644 index 0000000..9dc6325 --- /dev/null +++ b/src/core/env.mjs @@ -0,0 +1,13 @@ +import fs from 'node:fs'; + +export function loadDotenv(path = '.env') { + if (!fs.existsSync(path)) return; + const lines = fs.readFileSync(path, 'utf8').split(/\r?\n/); + for (const raw of lines) { + const line = raw.trim(); + if (!line || line.startsWith('#') || !line.includes('=')) continue; + const [key, ...rest] = line.split('='); + const value = rest.join('=').trim().replace(/^['"]|['"]$/g, ''); + if (!(key.trim() in process.env)) process.env[key.trim()] = value; + } +} diff --git a/src/core/event-envelope.mjs b/src/core/event-envelope.mjs new file mode 100644 index 0000000..632dc6a --- /dev/null +++ b/src/core/event-envelope.mjs @@ -0,0 +1,14 @@ +import crypto from 'node:crypto'; + +export function makeEventEnvelope({ venue, eventType, payload, raw = null, key = null }) { + return { + event_id: crypto.randomUUID(), + schema_version: 1, + venue, + event_type: eventType, + observed_at: new Date().toISOString(), + key, + payload, + raw, + }; +} diff --git a/src/core/log.mjs b/src/core/log.mjs new file mode 100644 index 0000000..5abc50a --- /dev/null +++ b/src/core/log.mjs @@ -0,0 +1,31 @@ +export function logStatus(message) { + const time = new Date().toISOString(); + console.error(`[${time}] ${message}`); +} + +export function startIdleHeartbeat({ + label, + getLastActivityAt, + getStatus, + idleAfterMs = 30_000, + checkEveryMs = 5_000, +}) { + let lastHeartbeatAt = 0; + + const timer = setInterval(() => { + const lastActivityAt = getLastActivityAt(); + const idleForMs = Date.now() - lastActivityAt; + + if (idleForMs < idleAfterMs) return; + if (Date.now() - lastHeartbeatAt < idleAfterMs) return; + + const seconds = Math.floor(idleForMs / 1000); + const suffix = getStatus ? `; ${getStatus()}` : ''; + logStatus(`${label} idle ${seconds}s${suffix}`); + lastHeartbeatAt = Date.now(); + }, checkEveryMs); + + if (typeof timer.unref === 'function') timer.unref(); + + return () => clearInterval(timer); +} diff --git a/src/core/pair-filter.mjs b/src/core/pair-filter.mjs new file mode 100644 index 0000000..172cab3 --- /dev/null +++ b/src/core/pair-filter.mjs @@ -0,0 +1,17 @@ +export function parsePairFilter(argv) { + const idx = argv.indexOf('--pair'); + if (idx === -1) return null; + const raw = argv[idx + 1]; + if (!raw || !raw.includes('->')) { + throw new Error("Use --pair 'asset_a->asset_b'"); + } + const [a, b] = raw.split('->').map((x) => x.trim().toLowerCase()); + return [a, b]; +} + +export function matchesPairFilter(assetIn, assetOut, pairFilter) { + if (!pairFilter) return true; + const x = assetIn.toLowerCase(); + const y = assetOut.toLowerCase(); + return (x === pairFilter[0] && y === pairFilter[1]) || (x === pairFilter[1] && y === pairFilter[0]); +} diff --git a/src/lib/config.mjs b/src/lib/config.mjs new file mode 100644 index 0000000..9d7b604 --- /dev/null +++ b/src/lib/config.mjs @@ -0,0 +1,33 @@ +import { loadDotenv } from './env.mjs'; + +const DEFAULTS = { + nearIntentsWsUrl: 'wss://solver-relay-v2.chaindefuser.com/ws', + kafkaBrokers: ['127.0.0.1:9092'], + kafkaClientId: 'trading-system', + kafkaTopicNormSwapDemand: 'norm.swap_demand', + kafkaConsumerGroupDummy: 'dummy-reactor-v1', +}; + +function splitCsv(value) { + return String(value || '') + .split(',') + .map((part) => part.trim()) + .filter(Boolean); +} + +export function loadConfig({ envPath = '.env' } = {}) { + loadDotenv(envPath); + + return { + nearIntentsApiKey: process.env.NEAR_INTENTS_API_KEY || '', + nearIntentsWsUrl: process.env.NEAR_INTENTS_WS_URL || DEFAULTS.nearIntentsWsUrl, + kafkaBrokers: splitCsv(process.env.KAFKA_BROKERS).length + ? splitCsv(process.env.KAFKA_BROKERS) + : DEFAULTS.kafkaBrokers, + kafkaClientId: process.env.KAFKA_CLIENT_ID || DEFAULTS.kafkaClientId, + kafkaTopicNormSwapDemand: + process.env.KAFKA_TOPIC_NORM_SWAP_DEMAND || DEFAULTS.kafkaTopicNormSwapDemand, + kafkaConsumerGroupDummy: + process.env.KAFKA_CONSUMER_GROUP_DUMMY || DEFAULTS.kafkaConsumerGroupDummy, + }; +} diff --git a/src/lib/env.mjs b/src/lib/env.mjs new file mode 100644 index 0000000..9dc6325 --- /dev/null +++ b/src/lib/env.mjs @@ -0,0 +1,13 @@ +import fs from 'node:fs'; + +export function loadDotenv(path = '.env') { + if (!fs.existsSync(path)) return; + const lines = fs.readFileSync(path, 'utf8').split(/\r?\n/); + for (const raw of lines) { + const line = raw.trim(); + if (!line || line.startsWith('#') || !line.includes('=')) continue; + const [key, ...rest] = line.split('='); + const value = rest.join('=').trim().replace(/^['"]|['"]$/g, ''); + if (!(key.trim() in process.env)) process.env[key.trim()] = value; + } +} diff --git a/src/lib/event-envelope.mjs b/src/lib/event-envelope.mjs new file mode 100644 index 0000000..445fb1a --- /dev/null +++ b/src/lib/event-envelope.mjs @@ -0,0 +1,37 @@ +export function buildEventEnvelope({ + source, + venue, + eventType, + eventId, + occurredAt = null, + ingestedAt = new Date(), + payload, +}) { + if (!source) throw new Error('Missing source'); + if (!venue) throw new Error('Missing venue'); + if (!eventType) throw new Error('Missing eventType'); + if (!eventId) throw new Error('Missing eventId'); + + const ingestedDate = parseDate(ingestedAt) ?? new Date(); + + return { + source: String(source), + venue: String(venue), + eventType: String(eventType), + eventId: String(eventId), + occurredAt: toIsoStringOrNull(occurredAt), + ingestedAt: ingestedDate.toISOString(), + payload, + }; +} + +function toIsoStringOrNull(value) { + const date = parseDate(value); + return date ? date.toISOString() : null; +} + +function parseDate(value) { + if (value == null) return null; + const date = value instanceof Date ? value : new Date(value); + return Number.isNaN(date.getTime()) ? null : date; +} diff --git a/src/venues/near-intents/ingest.mjs b/src/venues/near-intents/ingest.mjs new file mode 100644 index 0000000..a506822 --- /dev/null +++ b/src/venues/near-intents/ingest.mjs @@ -0,0 +1,5 @@ +import { startNearIntentsWs } from './ws.mjs'; + +export function startNearIntentsIngest(options) { + return startNearIntentsWs(options); +} diff --git a/src/venues/near-intents/normalize.mjs b/src/venues/near-intents/normalize.mjs new file mode 100644 index 0000000..fc8ab07 --- /dev/null +++ b/src/venues/near-intents/normalize.mjs @@ -0,0 +1,50 @@ +import { buildEventEnvelope } from '../../lib/event-envelope.mjs'; + +export function buildNearIntentsQuoteEnvelope(message, { ingestedAt = new Date() } = {}) { + const raw = isRecord(message) ? message : {}; + const payload = normalizeNearIntentsQuote(raw); + if (!payload) return null; + + const occurredAt = first(raw, ['created_at', 'createdAt', 'timestamp', 'ts']); + + return buildEventEnvelope({ + source: 'near-intents.ws', + venue: 'near-intents', + eventType: 'quote', + eventId: payload.quoteId, + occurredAt, + ingestedAt, + payload, + }); +} + +export function normalizeNearIntentsQuote(message) { + const quoteId = first(message, ['quote_id', 'quoteRequestId', 'request_id', 'id']); + const assetIn = first(message, ['defuse_asset_identifier_in', 'sellToken', 'asset_in']); + const assetOut = first(message, ['defuse_asset_identifier_out', 'buyToken', 'asset_out']); + if (!quoteId || !assetIn || !assetOut) return null; + + return { + quoteId: String(quoteId), + assetIn: String(assetIn), + assetOut: String(assetOut), + amountIn: stringify(first(message, ['exact_amount_in', 'sellAmount', 'amount_in'])), + amountOut: stringify(first(message, ['exact_amount_out', 'buyAmount', 'amount_out', 'expectedOut', 'quoted_amount_out'])), + ttlMs: stringify(first(message, ['min_deadline_ms', 'ttl_ms', 'deadline_ms'])), + }; +} + +function first(obj, keys) { + for (const key of keys) { + if (obj[key] != null) return obj[key]; + } + return null; +} + +function stringify(value) { + return value == null ? null : String(value); +} + +function isRecord(value) { + return !!value && typeof value === 'object' && !Array.isArray(value); +} diff --git a/src/venues/near-intents/ws.mjs b/src/venues/near-intents/ws.mjs new file mode 100644 index 0000000..8032a50 --- /dev/null +++ b/src/venues/near-intents/ws.mjs @@ -0,0 +1,162 @@ +import { matchesPairFilter } from '../../core/pair-filter.mjs'; +import { logStatus, startIdleHeartbeat } from '../../core/log.mjs'; +import { buildNearIntentsQuoteEnvelope } from './normalize.mjs'; + +const DEFAULT_WS_URL = 'wss://solver-relay-v2.chaindefuser.com/ws'; +const QUOTE_SUB_ID = 1; +const QUOTE_STATUS_SUB_ID = 2; + +export async function startNearIntentsWs({ + apiKey, + wsUrl = DEFAULT_WS_URL, + pairFilter, + producer, + topic, + onPublish = defaultOnPublish, +}) { + if (!apiKey) throw new Error('Missing NEAR_INTENTS_API_KEY'); + + let quoteSubscriptionId = null; + let quoteStatusSubscriptionId = null; + let lastStatusAt = Date.now(); + let publishedCount = 0; + let publishLocked = false; + + function connect() { + const ws = new WebSocket(wsUrl, { + headers: { Authorization: `Bearer ${apiKey}` }, + }); + + ws.addEventListener('open', () => { + logStatus('near-intents connected'); + ws.send(JSON.stringify({ jsonrpc: '2.0', id: QUOTE_SUB_ID, method: 'subscribe', params: ['quote'] })); + ws.send(JSON.stringify({ jsonrpc: '2.0', id: QUOTE_STATUS_SUB_ID, method: 'subscribe', params: ['quote_status'] })); + }); + + ws.addEventListener('message', async (event) => { + lastStatusAt = Date.now(); + const text = typeof event.data === 'string' ? event.data : Buffer.from(event.data).toString('utf8'); + + let payload; + try { + payload = JSON.parse(text); + } catch { + return; + } + + if (payload?.id === QUOTE_SUB_ID) { + quoteSubscriptionId = extractSubscriptionId(payload.result); + return; + } + + if (payload?.id === QUOTE_STATUS_SUB_ID) { + quoteStatusSubscriptionId = extractSubscriptionId(payload.result); + return; + } + + const eventFrame = extractQuoteEventFrame(payload); + if (!eventFrame) return; + + const { subscription, merged } = eventFrame; + + if (quoteStatusSubscriptionId && subscription === quoteStatusSubscriptionId) return; + if (quoteSubscriptionId && subscription && subscription !== quoteSubscriptionId) return; + if (publishLocked) return; + + const envelope = buildNearIntentsQuoteEnvelope(merged); + if (!envelope) return; + + const assetIn = envelope.payload?.assetIn; + const assetOut = envelope.payload?.assetOut; + if (!assetIn || !assetOut) return; + if (!matchesPairFilter(assetIn, assetOut, pairFilter)) return; + + publishLocked = true; + try { + await producer.sendJson(topic, envelope, { key: envelope.eventId }); + publishedCount += 1; + onPublish(envelope, publishedCount); + } catch (error) { + logStatus(`kafka publish failed: ${error.message || 'unknown error'}`); + } finally { + publishLocked = false; + } + }); + + ws.addEventListener('close', () => { + logStatus('near-intents disconnected; reconnecting in 2s'); + setTimeout(connect, 2000); + }); + + ws.addEventListener('error', (err) => { + logStatus(`near-intents socket error: ${err.message || 'unknown error'}`); + }); + } + + startIdleHeartbeat({ + label: 'near-intents', + getLastActivityAt: () => lastStatusAt, + getStatus: () => `published=${publishedCount}`, + }); + + connect(); +} + +function extractSubscriptionId(result) { + if (typeof result === 'string') return result; + if (result && typeof result === 'object') { + return result.subscription || result.subscription_id || result.subscriber_id || null; + } + return null; +} + +function extractQuoteEventFrame(payload) { + const candidates = []; + + if (payload?.method === 'event' && payload?.params) { + candidates.push(payload.params); + } + + if (payload?.result && typeof payload.result === 'object') { + candidates.push(payload.result); + } + + if (payload && typeof payload === 'object') { + candidates.push(payload); + } + + for (const candidate of candidates) { + const data = candidate?.data; + const metadata = candidate?.metadata; + const merged = isRecord(data) || isRecord(metadata) + ? { ...(isRecord(metadata) ? metadata : {}), ...(isRecord(data) ? data : {}) } + : candidate; + + if (!isRecord(merged)) continue; + if (!looksLikeQuotePayload(merged)) continue; + + return { + subscription: candidate?.subscription ?? null, + merged, + }; + } + + return null; +} + +function looksLikeQuotePayload(payload) { + return Boolean( + payload.quote_hash + || payload.quote_id + || payload.defuse_asset_identifier_in + || payload.defuse_asset_identifier_out + || payload.asset_in + || payload.asset_out, + ); +} + +function isRecord(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value); +} + +function defaultOnPublish() {}