From 02435960a86e15f451fb92f3cd64e51ad41776ee Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 12 Oct 2022 14:46:58 -0700 Subject: [PATCH 1/3] Allow to configure log level from Python --- pulsar/__init__.py | 45 +++++++++++++++++++++++++++++++++++++++++--- src/config.cc | 16 +++++++++++++++- src/enums.cc | 6 ++++++ tests/pulsar_test.py | 24 +++++++++++++++++++++++ 4 files changed, 87 insertions(+), 4 deletions(-) diff --git a/pulsar/__init__.py b/pulsar/__init__.py index 942ec8f..cbfb680 100644 --- a/pulsar/__init__.py +++ b/pulsar/__init__.py @@ -103,7 +103,8 @@ def send_callback(res, msg_id): import logging import _pulsar -from _pulsar import Result, CompressionType, ConsumerType, InitialPosition, PartitionsRoutingMode, BatchingType # noqa: F401 +from _pulsar import Result, CompressionType, ConsumerType, InitialPosition, PartitionsRoutingMode, BatchingType, \ + LoggerLevel # noqa: F401 from pulsar.exceptions import * @@ -448,7 +449,6 @@ def __init__(self, service_url, _check_type_or_none(str, tls_trust_certs_file_path, 'tls_trust_certs_file_path') _check_type(bool, tls_allow_insecure_connection, 'tls_allow_insecure_connection') _check_type(bool, tls_validate_hostname, 'tls_validate_hostname') - _check_type_or_none(logging.Logger, logger, 'logger') _check_type_or_none(str, listener_name, 'listener_name') conf = _pulsar.ClientConfiguration() @@ -461,7 +461,16 @@ def __init__(self, service_url, conf.concurrent_lookup_requests(concurrent_lookup_requests) if log_conf_file_path: conf.log_conf_file_path(log_conf_file_path) - conf.set_logger(self._prepare_logger(logger) if logger else None) + + if isinstance(logger, logging.Logger): + conf.set_logger(self._prepare_logger(logger)) + elif isinstance(logger, ConsoleLogger): + conf.set_console_logger(logger.log_level) + elif isinstance(logger, FileLogger): + conf.set_file_logger(logger.log_level, logger.log_file) + elif logger is not None: + raise ValueError("Logger is expected to be either None, logger.Logger, pulsar.ConsoleLogger or pulsar.FileLogger") + if listener_name: conf.listener_name(listener_name) if use_tls or service_url.startswith('pulsar+ssl://') or service_url.startswith('https://'): @@ -1405,6 +1414,36 @@ def __init__(self, public_key_path, private_key_path): _check_type(str, private_key_path, 'private_key_path') self.cryptoKeyReader = _pulsar.CryptoKeyReader(public_key_path, private_key_path) + +class ConsoleLogger: + """ + Logger that writes on standard output + + **Args** + + * `log_level`: The logging level. eg: `pulsar.LoggerLevel.Info` + """ + def __init__(self, log_level=_pulsar.LoggerLevel.Info): + _check_type(_pulsar.LoggerLevel, log_level, 'log_level') + self.log_level = log_level + + +class FileLogger: + """ + Logger that writes into a file + + **Args** + + * `log_level`: The logging level. eg: `pulsar.LoggerLevel.Info` + * `log_file`: The file where to write the logs + """ + def __init__(self, log_level, log_file): + _check_type(_pulsar.LoggerLevel, log_level, 'log_level') + _check_type(str, log_file, 'log_file') + self.log_level = log_level + self.log_file = log_file + + def _check_type(var_type, var, name): if not isinstance(var, var_type): raise ValueError("Argument %s is expected to be of type '%s' and not '%s'" diff --git a/src/config.cc b/src/config.cc index c312648..3a9c14b 100644 --- a/src/config.cc +++ b/src/config.cc @@ -162,6 +162,17 @@ static ClientConfiguration& ClientConfiguration_setLogger(ClientConfiguration& c return conf; } +static ClientConfiguration& ClientConfiguration_setConsoleLogger(ClientConfiguration& conf, Logger::Level level) { + conf.setLogger(new ConsoleLoggerFactory(level)); + return conf; +} + +static ClientConfiguration& ClientConfiguration_setFileLogger(ClientConfiguration& conf, Logger::Level level, + const std::string& logFile) { + conf.setLogger(new FileLoggerFactory(level, logFile)); + return conf; +} + void export_config() { using namespace boost::python; @@ -190,7 +201,10 @@ void export_config() { return_self<>()) .def("tls_validate_hostname", &ClientConfiguration::setValidateHostName, return_self<>()) .def("listener_name", &ClientConfiguration::setListenerName, return_self<>()) - .def("set_logger", &ClientConfiguration_setLogger, return_self<>()); + .def("set_logger", &ClientConfiguration_setLogger, return_self<>()) + .def("set_console_logger", &ClientConfiguration_setConsoleLogger, return_self<>()) + .def("set_file_logger", &ClientConfiguration_setFileLogger, return_self<>()); + class_("ProducerConfiguration") .def("producer_name", &ProducerConfiguration::getProducerName, diff --git a/src/enums.cc b/src/enums.cc index 92f08a1..733ee3e 100644 --- a/src/enums.cc +++ b/src/enums.cc @@ -111,4 +111,10 @@ void export_enums() { enum_("BatchingType", "Supported batching types") .value("Default", ProducerConfiguration::DefaultBatching) .value("KeyBased", ProducerConfiguration::KeyBasedBatching); + + enum_("LoggerLevel") + .value("Debug", Logger::LEVEL_DEBUG) + .value("Info", Logger::LEVEL_INFO) + .value("Warn", Logger::LEVEL_WARN) + .value("Error", Logger::LEVEL_ERROR); } diff --git a/tests/pulsar_test.py b/tests/pulsar_test.py index 86abbfe..e3e0ede 100755 --- a/tests/pulsar_test.py +++ b/tests/pulsar_test.py @@ -1238,6 +1238,30 @@ def test_json_schema_encode(self): second_encode = schema.encode(record) self.assertEqual(first_encode, second_encode) + def configure_log_level(self): + client = pulsar.Client( + service_url="pulsar://localhost:6650", + logger=pulsar.ConsoleLogger(pulsar.LoggerLevel.Debug) + ) + + producer = client.create_producer( + topic='test_log_level' + ) + + producer.send(b'hello') + + def configure_log_to_file(self): + client = pulsar.Client( + service_url="pulsar://localhost:6650", + logger=pulsar.FileLogger(pulsar.LoggerLevel.Debug, 'test.log') + ) + + producer = client.create_producer( + topic='test_log_to_file' + ) + + producer.send(b'hello') + def test_logger_thread_leaks(self): def _do_connect(close): logger = logging.getLogger(str(threading.current_thread().ident)) From 9cecca9d51251a854f88d6076d40ca76553911d4 Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Wed, 12 Oct 2022 15:03:04 -0700 Subject: [PATCH 2/3] Fixed test method names --- tests/pulsar_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pulsar_test.py b/tests/pulsar_test.py index e3e0ede..0177b34 100755 --- a/tests/pulsar_test.py +++ b/tests/pulsar_test.py @@ -1238,7 +1238,7 @@ def test_json_schema_encode(self): second_encode = schema.encode(record) self.assertEqual(first_encode, second_encode) - def configure_log_level(self): + def test_configure_log_level(self): client = pulsar.Client( service_url="pulsar://localhost:6650", logger=pulsar.ConsoleLogger(pulsar.LoggerLevel.Debug) @@ -1250,7 +1250,7 @@ def configure_log_level(self): producer.send(b'hello') - def configure_log_to_file(self): + def test_configure_log_to_file(self): client = pulsar.Client( service_url="pulsar://localhost:6650", logger=pulsar.FileLogger(pulsar.LoggerLevel.Debug, 'test.log') From ea57ff370cf4efeac543b0d3ef81f7e3eed4195e Mon Sep 17 00:00:00 2001 From: Matteo Merli Date: Thu, 13 Oct 2022 10:40:11 -0700 Subject: [PATCH 3/3] Candidate-1 was deleted.. use candidate-2 --- build-support/install-cpp-client.sh | 2 +- pkg/mac/build-pulsar-cpp.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-support/install-cpp-client.sh b/build-support/install-cpp-client.sh index ac493c1..4c4608f 100755 --- a/build-support/install-cpp-client.sh +++ b/build-support/install-cpp-client.sh @@ -34,7 +34,7 @@ cd /tmp # Fetch the client binaries ## TODO: Fetch from official release once it's available -BASE_URL=https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-cpp-${CPP_CLIENT_VERSION}-candidate-1 +BASE_URL=https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-cpp-${CPP_CLIENT_VERSION}-candidate-2 UNAME_ARCH=$(uname -m) if [ $UNAME_ARCH == 'aarch64' ]; then diff --git a/pkg/mac/build-pulsar-cpp.sh b/pkg/mac/build-pulsar-cpp.sh index c1eb514..ccf7eb4 100755 --- a/pkg/mac/build-pulsar-cpp.sh +++ b/pkg/mac/build-pulsar-cpp.sh @@ -38,7 +38,7 @@ DEPS_PREFIX=${CACHE_DIR_DEPS}/install ############################################################################### ## TODO: Fetch from official release -curl -O -L https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-cpp-${PULSAR_CPP_VERSION}-candidate-1/apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz +curl -O -L https://dist.apache.org/repos/dist/dev/pulsar/pulsar-client-cpp-${PULSAR_CPP_VERSION}-candidate-2/apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz tar xfz apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}.tar.gz if [ ! -f apache-pulsar-client-cpp-${PULSAR_CPP_VERSION}/.done ]; then