Skip to content

setuptools_betterproto

setuptools_betterproto ¤

A modern setuptools plugin to generate Python files from proto files using betterproto.

Classes¤

setuptools_betterproto.AddProtoFiles ¤

Bases: BaseProtoCommand

A command to add the proto files to the source distribution.

Source code in setuptools_betterproto/_command.py 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
class AddProtoFiles(BaseProtoCommand):
    """A command to add the proto files to the source distribution."""

    def run(self) -> None:
        """Copy the proto files to the source distribution."""
        proto_files = self.config.expanded_proto_files
        include_files = self.config.expanded_include_files

        if include_files and not proto_files:
            _logger.warning(
                "Some proto files (%s) were found in the `include_paths` (%s), but "
                "no proto files were found in the `proto_path`. You probably want to "
                "check if your `proto_path` (%s) and `proto_glob` (%s) are configured "
                "correctly. We are not adding the found include files to the source "
                "distribution automatically!",
                len(include_files),
                ", ".join(self.config.include_paths),
                self.config.proto_path,
                self.config.proto_glob,
            )
            return

        if not proto_files:
            _logger.warning(
                "No proto files were found in the `proto_path` (%s) using `proto_glob` "
                "(%s). You probably want to check if you `proto_path` and `proto_glob` "
                "are configured correctly. We are not adding any proto files to the "
                "source distribution automatically!",
                self.config.proto_path,
                self.config.proto_glob,
            )
            return

        dest_dir = self.distribution.get_fullname()

        for file in (*proto_files, *include_files):
            self.copy_with_directories(file, os.path.join(dest_dir, file))

        _logger.info("added %s proto files", len(proto_files) + len(include_files))

    def copy_with_directories(self, src: str, dest: str) -> None:
        """Copy a file from src to dest, creating the destination's directory tree.

        Any directories that do not exist in dest will be created.

        Args:
            src: The full path of the source file.
            dest: The full path of the destination file.
        """
        dest_dir = os.path.dirname(dest)
        if not os.path.exists(dest_dir):
            _logger.debug("creating directory %s", dest_dir)
            os.makedirs(dest_dir)
        _logger.info("adding proto file to %s", dest)
        shutil.copyfile(src, dest)
Attributes¤
config instance-attribute ¤
config: ProtobufConfig

The configuration object for the command.

description class-attribute instance-attribute ¤
description: str = (
    "compile protobuf files using betterproto"
)

Description of the command.

include_paths instance-attribute ¤
include_paths: str

Comma-separated list of paths to include when compiling the protobuf files.

out_path instance-attribute ¤
out_path: str

The path of the root directory where the Python files will be generated.

proto_glob instance-attribute ¤
proto_glob: str

The glob pattern to use to find the protobuf files.

proto_path instance-attribute ¤
proto_path: str

The path of the root directory containing the protobuf files.

user_options class-attribute instance-attribute ¤
user_options: list[tuple[str, str | None, str]] = [
    (
        "proto-path=",
        None,
        "path of the root directory containing the protobuf files",
    ),
    (
        "proto-glob=",
        None,
        "glob pattern to use to find the protobuf files",
    ),
    (
        "include-paths=",
        None,
        "comma-separated list of paths to include when compiling the protobuf files",
    ),
    (
        "out-dir=",
        None,
        "path of the root directory where the Python files will be generated",
    ),
]

Options of the command.

Functions¤
copy_with_directories ¤
copy_with_directories(src: str, dest: str) -> None

Copy a file from src to dest, creating the destination's directory tree.

Any directories that do not exist in dest will be created.

PARAMETER DESCRIPTION
src

The full path of the source file.

TYPE: str

dest

The full path of the destination file.

TYPE: str

Source code in setuptools_betterproto/_command.py 
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
def copy_with_directories(self, src: str, dest: str) -> None:
    """Copy a file from src to dest, creating the destination's directory tree.

    Any directories that do not exist in dest will be created.

    Args:
        src: The full path of the source file.
        dest: The full path of the destination file.
    """
    dest_dir = os.path.dirname(dest)
    if not os.path.exists(dest_dir):
        _logger.debug("creating directory %s", dest_dir)
        os.makedirs(dest_dir)
    _logger.info("adding proto file to %s", dest)
    shutil.copyfile(src, dest)
finalize_options ¤
finalize_options() -> None

Finalize options by converting them to a ProtobufConfig object.

Source code in setuptools_betterproto/_command.py 
80
81
82
83
84
85
86
87
88
@override
def finalize_options(self) -> None:
    """Finalize options by converting them to a ProtobufConfig object."""
    self.config = _config.ProtobufConfig.from_strings(
        proto_path=self.proto_path,
        proto_glob=self.proto_glob,
        include_paths=self.include_paths,
        out_path=self.out_path,
    )
initialize_options ¤
initialize_options() -> None

Initialize options with default values.

Source code in setuptools_betterproto/_command.py 
70
71
72
73
74
75
76
77
78
@override
def initialize_options(self) -> None:
    """Initialize options with default values."""
    self.config = _config.ProtobufConfig.from_pyproject_toml()

    self.proto_path = self.config.proto_path
    self.proto_glob = self.config.proto_glob
    self.include_paths = ",".join(self.config.include_paths)
    self.out_path = self.config.out_path
run ¤
run() -> None

Copy the proto files to the source distribution.

Source code in setuptools_betterproto/_command.py 
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
def run(self) -> None:
    """Copy the proto files to the source distribution."""
    proto_files = self.config.expanded_proto_files
    include_files = self.config.expanded_include_files

    if include_files and not proto_files:
        _logger.warning(
            "Some proto files (%s) were found in the `include_paths` (%s), but "
            "no proto files were found in the `proto_path`. You probably want to "
            "check if your `proto_path` (%s) and `proto_glob` (%s) are configured "
            "correctly. We are not adding the found include files to the source "
            "distribution automatically!",
            len(include_files),
            ", ".join(self.config.include_paths),
            self.config.proto_path,
            self.config.proto_glob,
        )
        return

    if not proto_files:
        _logger.warning(
            "No proto files were found in the `proto_path` (%s) using `proto_glob` "
            "(%s). You probably want to check if you `proto_path` and `proto_glob` "
            "are configured correctly. We are not adding any proto files to the "
            "source distribution automatically!",
            self.config.proto_path,
            self.config.proto_glob,
        )
        return

    dest_dir = self.distribution.get_fullname()

    for file in (*proto_files, *include_files):
        self.copy_with_directories(file, os.path.join(dest_dir, file))

    _logger.info("added %s proto files", len(proto_files) + len(include_files))

setuptools_betterproto.CompileBetterproto ¤

Bases: BaseProtoCommand

A command to compile the protobuf files.

Source code in setuptools_betterproto/_command.py 
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
class CompileBetterproto(BaseProtoCommand):
    """A command to compile the protobuf files."""

    @override
    def run(self) -> None:
        """Compile the protobuf files to Python."""
        proto_files = self.config.expanded_proto_files

        if not proto_files:
            _logger.warning(
                "No proto files were found in the `proto_path` (%s) using `proto_glob` "
                "(%s). You probably want to check if you `proto_path` and `proto_glob` "
                "are configured correctly. We are not compiling any proto files!",
                self.config.proto_path,
                self.config.proto_glob,
            )
            return

        protoc_cmd = [
            sys.executable,
            "-m",
            "grpc_tools.protoc",
            *(f"-I{p}" for p in [self.config.proto_path, *self.config.include_paths]),
            f"--python_betterproto_out={self.config.out_path}",
            *proto_files,
        ]

        _logger.info("compiling proto files via: %s", " ".join(protoc_cmd))
        subprocess.run(protoc_cmd, check=True)
Attributes¤
config instance-attribute ¤
config: ProtobufConfig

The configuration object for the command.

description class-attribute instance-attribute ¤
description: str = (
    "compile protobuf files using betterproto"
)

Description of the command.

include_paths instance-attribute ¤
include_paths: str

Comma-separated list of paths to include when compiling the protobuf files.

out_path instance-attribute ¤
out_path: str

The path of the root directory where the Python files will be generated.

proto_glob instance-attribute ¤
proto_glob: str

The glob pattern to use to find the protobuf files.

proto_path instance-attribute ¤
proto_path: str

The path of the root directory containing the protobuf files.

user_options class-attribute instance-attribute ¤
user_options: list[tuple[str, str | None, str]] = [
    (
        "proto-path=",
        None,
        "path of the root directory containing the protobuf files",
    ),
    (
        "proto-glob=",
        None,
        "glob pattern to use to find the protobuf files",
    ),
    (
        "include-paths=",
        None,
        "comma-separated list of paths to include when compiling the protobuf files",
    ),
    (
        "out-dir=",
        None,
        "path of the root directory where the Python files will be generated",
    ),
]

Options of the command.

Functions¤
finalize_options ¤
finalize_options() -> None

Finalize options by converting them to a ProtobufConfig object.

Source code in setuptools_betterproto/_command.py 
80
81
82
83
84
85
86
87
88
@override
def finalize_options(self) -> None:
    """Finalize options by converting them to a ProtobufConfig object."""
    self.config = _config.ProtobufConfig.from_strings(
        proto_path=self.proto_path,
        proto_glob=self.proto_glob,
        include_paths=self.include_paths,
        out_path=self.out_path,
    )
initialize_options ¤
initialize_options() -> None

Initialize options with default values.

Source code in setuptools_betterproto/_command.py 
70
71
72
73
74
75
76
77
78
@override
def initialize_options(self) -> None:
    """Initialize options with default values."""
    self.config = _config.ProtobufConfig.from_pyproject_toml()

    self.proto_path = self.config.proto_path
    self.proto_glob = self.config.proto_glob
    self.include_paths = ",".join(self.config.include_paths)
    self.out_path = self.config.out_path
run ¤
run() -> None

Compile the protobuf files to Python.

Source code in setuptools_betterproto/_command.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
@override
def run(self) -> None:
    """Compile the protobuf files to Python."""
    proto_files = self.config.expanded_proto_files

    if not proto_files:
        _logger.warning(
            "No proto files were found in the `proto_path` (%s) using `proto_glob` "
            "(%s). You probably want to check if you `proto_path` and `proto_glob` "
            "are configured correctly. We are not compiling any proto files!",
            self.config.proto_path,
            self.config.proto_glob,
        )
        return

    protoc_cmd = [
        sys.executable,
        "-m",
        "grpc_tools.protoc",
        *(f"-I{p}" for p in [self.config.proto_path, *self.config.include_paths]),
        f"--python_betterproto_out={self.config.out_path}",
        *proto_files,
    ]

    _logger.info("compiling proto files via: %s", " ".join(protoc_cmd))
    subprocess.run(protoc_cmd, check=True)

setuptools_betterproto.ProtobufConfig dataclass ¤

A configuration for the protobuf files.

The configuration can be loaded from the pyproject.toml file using the class method from_pyproject_toml().

Source code in setuptools_betterproto/_config.py 
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
@dataclasses.dataclass(frozen=True, kw_only=True)
class ProtobufConfig:
    """A configuration for the protobuf files.

    The configuration can be loaded from the `pyproject.toml` file using the class
    method `from_pyproject_toml()`.
    """

    proto_path: str = "."
    """The path of the root directory containing the protobuf files."""

    proto_glob: str = "*.proto"
    """The glob pattern to use to find the protobuf files."""

    include_paths: Sequence[str] = ()
    """The paths to add to the include path when compiling the protobuf files."""

    out_path: str = "."
    """The path of the root directory where the Python files will be generated."""

    @classmethod
    def from_pyproject_toml(
        cls, path: str = "pyproject.toml", /, **defaults: Any
    ) -> Self:
        """Create a new configuration by loading the options from a `pyproject.toml` file.

        The options are read from the `[tool.frequenz-repo-config.protobuf]`
        section of the `pyproject.toml` file.

        Args:
            path: The path to the `pyproject.toml` file.
            **defaults: The default values for the options missing in the file.  If
                a default is missing too, then the default in this class will be used.

        Returns:
            The configuration.
        """
        try:
            with open(path, "rb") as toml_file:
                pyproject_toml = tomllib.load(toml_file)
        except FileNotFoundError:
            return cls(**defaults)
        except OSError as err:
            _logger.warning("WARNING: Failed to load pyproject.toml: %s", err)
            return cls(**defaults)

        try:
            config = pyproject_toml["tool"]["setuptools_betterproto"]
        except KeyError:
            return cls(**defaults)

        default = cls(**defaults)
        known_keys = frozenset(dataclasses.asdict(default).keys())
        config_keys = frozenset(config.keys())
        if unknown_keys := config_keys - known_keys:
            _logger.warning(
                "WARNING: There are some configuration keys in pyproject.toml we don't "
                "know about and will be ignored: %s",
                ", ".join(f"'{k}'" for k in unknown_keys),
            )

        attrs = dict(defaults, **{k: config[k] for k in (known_keys & config_keys)})
        return dataclasses.replace(default, **attrs)

    @classmethod
    def from_strings(
        cls, *, proto_path: str, proto_glob: str, include_paths: str, out_path: str
    ) -> Self:
        """Create a new configuration from plain strings.

        Args:
            proto_path: The path of the root directory containing the protobuf files.
            proto_glob: The glob pattern to use to find the protobuf files.
            include_paths: The paths to add to the include path when compiling the
                protobuf files.
            out_path: The path of the root directory where the Python files will be
                generated.

        Returns:
            The configuration.
        """
        return cls(
            proto_path=proto_path,
            proto_glob=proto_glob,
            include_paths=[p.strip() for p in filter(None, include_paths.split(","))],
            out_path=out_path,
        )

    @property
    def expanded_proto_files(self) -> list[str]:
        """The files in the `proto_path` expanded according to the configured glob."""
        proto_path = pathlib.Path(self.proto_path)
        return [str(proto_file) for proto_file in proto_path.rglob(self.proto_glob)]

    @property
    def expanded_include_files(self) -> list[str]:
        """The files in the `include_paths` expanded according to the configured glob."""
        return [
            str(proto_file)
            for include_path in map(pathlib.Path, self.include_paths)
            for proto_file in include_path.rglob(self.proto_glob)
        ]
Attributes¤
expanded_include_files property ¤
expanded_include_files: list[str]

The files in the include_paths expanded according to the configured glob.

expanded_proto_files property ¤
expanded_proto_files: list[str]

The files in the proto_path expanded according to the configured glob.

include_paths class-attribute instance-attribute ¤
include_paths: Sequence[str] = ()

The paths to add to the include path when compiling the protobuf files.

out_path class-attribute instance-attribute ¤
out_path: str = '.'

The path of the root directory where the Python files will be generated.

proto_glob class-attribute instance-attribute ¤
proto_glob: str = '*.proto'

The glob pattern to use to find the protobuf files.

proto_path class-attribute instance-attribute ¤
proto_path: str = '.'

The path of the root directory containing the protobuf files.

Functions¤
from_pyproject_toml classmethod ¤
from_pyproject_toml(
    path: str = "pyproject.toml", /, **defaults: Any
) -> Self

Create a new configuration by loading the options from a pyproject.toml file.

The options are read from the [tool.frequenz-repo-config.protobuf] section of the pyproject.toml file.

PARAMETER DESCRIPTION
path

The path to the pyproject.toml file.

TYPE: str DEFAULT: 'pyproject.toml'

**defaults

The default values for the options missing in the file. If a default is missing too, then the default in this class will be used.

TYPE: Any DEFAULT: {}

RETURNS DESCRIPTION
Self

The configuration.
Source code in setuptools_betterproto/_config.py 
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
@classmethod
def from_pyproject_toml(
    cls, path: str = "pyproject.toml", /, **defaults: Any
) -> Self:
    """Create a new configuration by loading the options from a `pyproject.toml` file.

    The options are read from the `[tool.frequenz-repo-config.protobuf]`
    section of the `pyproject.toml` file.

    Args:
        path: The path to the `pyproject.toml` file.
        **defaults: The default values for the options missing in the file.  If
            a default is missing too, then the default in this class will be used.

    Returns:
        The configuration.
    """
    try:
        with open(path, "rb") as toml_file:
            pyproject_toml = tomllib.load(toml_file)
    except FileNotFoundError:
        return cls(**defaults)
    except OSError as err:
        _logger.warning("WARNING: Failed to load pyproject.toml: %s", err)
        return cls(**defaults)

    try:
        config = pyproject_toml["tool"]["setuptools_betterproto"]
    except KeyError:
        return cls(**defaults)

    default = cls(**defaults)
    known_keys = frozenset(dataclasses.asdict(default).keys())
    config_keys = frozenset(config.keys())
    if unknown_keys := config_keys - known_keys:
        _logger.warning(
            "WARNING: There are some configuration keys in pyproject.toml we don't "
            "know about and will be ignored: %s",
            ", ".join(f"'{k}'" for k in unknown_keys),
        )

    attrs = dict(defaults, **{k: config[k] for k in (known_keys & config_keys)})
    return dataclasses.replace(default, **attrs)
from_strings classmethod ¤
from_strings(
    *,
    proto_path: str,
    proto_glob: str,
    include_paths: str,
    out_path: str
) -> Self

Create a new configuration from plain strings.

PARAMETER DESCRIPTION
proto_path

The path of the root directory containing the protobuf files.

TYPE: str

proto_glob

The glob pattern to use to find the protobuf files.

TYPE: str

include_paths

The paths to add to the include path when compiling the protobuf files.

TYPE: str

out_path

The path of the root directory where the Python files will be generated.

TYPE: str

RETURNS DESCRIPTION
Self

The configuration.
Source code in setuptools_betterproto/_config.py 
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
@classmethod
def from_strings(
    cls, *, proto_path: str, proto_glob: str, include_paths: str, out_path: str
) -> Self:
    """Create a new configuration from plain strings.

    Args:
        proto_path: The path of the root directory containing the protobuf files.
        proto_glob: The glob pattern to use to find the protobuf files.
        include_paths: The paths to add to the include path when compiling the
            protobuf files.
        out_path: The path of the root directory where the Python files will be
            generated.

    Returns:
        The configuration.
    """
    return cls(
        proto_path=proto_path,
        proto_glob=proto_glob,
        include_paths=[p.strip() for p in filter(None, include_paths.split(","))],
        out_path=out_path,
    )

Functions¤

setuptools_betterproto.finalize_distribution_options ¤

finalize_distribution_options(dist: Distribution) -> None

Make some final adjustments to the distribution options.

We need to do some stuff early when setuptools runs to make sure all files are compiled and distributed appropriately.

  1. Replace the sdist command with a custom one that includes the proto files.
  2. Add the compile_betterproto command to the build sub-commands.
  3. If the distribution is a binary distribution, build the proto files early.
PARAMETER DESCRIPTION
dist

The distribution object.

TYPE: Distribution

Source code in setuptools_betterproto/_install.py 
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
def finalize_distribution_options(dist: Distribution) -> None:
    """Make some final adjustments to the distribution options.

    We need to do some stuff early when setuptools runs to make sure all files are
    compiled and distributed appropriately.

    1. Replace the sdist command with a custom one that includes the proto files.
    2. Add the `compile_betterproto` command to the build sub-commands.
    3. If the distribution is a binary distribution, build the proto files early.

    Args:
        dist: The distribution object.
    """
    config = _config.ProtobufConfig.from_pyproject_toml()
    replace_sdist_command(dist)
    add_build_subcommand_compile_betterproto(dist)

    if not building_bdist(dist):
        return

    if not config.expanded_proto_files:
        _logger.warning(
            "No proto files found in %s with glob %s, skipping early automatic "
            "compilation of proto files.",
            config.proto_path,
            config.proto_glob,
        )
        return

    build_proto(dist)