edgedb 是基于python开发的，同时集成了cython 以下为包的setup.py 配置，从里面我们可以看到关于edgedb 的一些依赖
以及构建过程
setup.py 源码
整体配置不算很多，500 多行，主要是cython extension 配置以及pg 构建配置，以及pg extension 配置,其中添加了关于pg 以及
pg 扩展build 的自定义cmdclass
- 代码
#
# This source file is part of the EdgeDB open source project.
#
# Copyright 2008-present MagicStack Inc. and the EdgeDB authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os.path
import pathlib
import platform
import shutil
import subprocess
import textwrap
from distutils import extension as distutils_extension
from distutils.command import build as distutils_build
from distutils.command import build_ext as distutils_build_ext
import setuptools
from setuptools.command import develop as setuptools_develop
RUNTIME_DEPS = [
'asyncpg~=0.18.2',
'click~=6.7',
'httptools>=0.0.13',
'immutables>=0.9',
'parsing~=1.6.1',
'prompt_toolkit>=2.0.0',
'psutil~=5.6.1',
'Pygments~=2.3.0',
'setproctitle~=1.1.10',
'setuptools_scm~=3.2.0',
'uvloop~=0.12.2',
'graphql-core~=2.1.0',
'promise~=2.2.0',
]
CYTHON_DEPENDENCY = 'Cython==0.29.6'
DOCS_DEPS = [
'Sphinx~=2.0.0',
'lxml~=4.2.5',
'sphinxcontrib-asyncio~=0.2.0',
]
BUILD_DEPS = [
CYTHON_DEPENDENCY,
]
EXTRA_DEPS = {
'test': [
'flake8~=3.7.5',
'pycodestyle~=2.5.0',
'coverage~=4.5.2',
'requests-xml~=0.2.3',
'lxml',
'requests-xml',
] + DOCS_DEPS,
'docs': DOCS_DEPS,
}
EXT_CFLAGS = ['-O2']
EXT_LDFLAGS = []
if platform.uname().system != 'Windows':
EXT_CFLAGS.extend([
'-std=c99', '-fsigned-char', '-Wall', '-Wsign-compare', '-Wconversion'
])
def _compile_parsers(build_lib, inplace=False):
import parsing
import edb.edgeql.parser.grammar.single as edgeql_spec
import edb.edgeql.parser.grammar.block as edgeql_spec2
import edb.edgeql.parser.grammar.sdldocument as schema_spec
base_path = pathlib.Path(__file__).parent.resolve()
for spec in (edgeql_spec, edgeql_spec2, schema_spec):
spec_path = pathlib.Path(spec.__file__).parent
subpath = pathlib.Path(str(spec_path)[len(str(base_path)) + 1:])
pickle_name = spec.__name__.rpartition('.')[2] + '.pickle'
pickle_path = subpath / pickle_name
cache = build_lib / pickle_path
cache.parent.mkdir(parents=True, exist_ok=True)
parsing.Spec(spec, pickleFile=str(cache), verbose=True)
if inplace:
shutil.copy2(cache, base_path / pickle_path)
def _compile_build_meta(build_lib, version, pg_config, runstatedir):
import pkg_resources
from edb.server import buildmeta
parsed_version = buildmeta.parse_version(
pkg_resources.parse_version(version))
vertuple = list(parsed_version._asdict().values())
vertuple[2] = int(vertuple[2])
vertuple = tuple(vertuple)
content = textwrap.dedent('''\
#
# This source file is part of the EdgeDB open source project.
#
# Copyright 2008-present MagicStack Inc. and the EdgeDB authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
#
# THIS FILE HAS BEEN AUTOMATICALLY GENERATED.
#
PG_CONFIG_PATH = {pg_config!r}
RUNSTATE_DIR = {runstatedir!r}
VERSION = {version!r}
''').format(version=vertuple, pg_config=pg_config, runstatedir=runstatedir)
directory = build_lib / 'edb' / 'server'
if not directory.exists():
directory.mkdir(parents=True)
with open(directory / '_buildmeta.py', 'w+t') as f:
f.write(content)
def _compile_postgres(build_base, *,
force_build=False, fresh_build=True,
run_configure=True, build_contrib=True):
proc = subprocess.run(
['git', 'submodule', 'status', 'postgres'],
stdout=subprocess.PIPE, universal_newlines=True, check=True)
status = proc.stdout
if status[0] == '-':
print(
'postgres submodule not initialized, '
'run `git submodule init; git submodule update`')
exit(1)
proc = subprocess.run(
['git', 'submodule', 'status', 'postgres'],
stdout=subprocess.PIPE, universal_newlines=True, check=True)
revision, _, _ = proc.stdout[1:].partition(' ')
source_stamp = proc.stdout[0] + revision
postgres_build = (build_base / 'postgres').resolve()
postgres_src = (pathlib.Path(__file__).parent / 'postgres').resolve()
postgres_build_stamp = postgres_build / 'stamp'
if postgres_build_stamp.exists():
with open(postgres_build_stamp, 'r') as f:
build_stamp = f.read()
else:
build_stamp = None
is_outdated = source_stamp != build_stamp
if is_outdated or force_build:
system = platform.system()
if system == 'Darwin':
uuidlib = 'e2fs'
elif system == 'Linux':
uuidlib = 'e2fs'
else:
raise NotImplementedError('unsupported system: {}'.format(system))
if fresh_build and postgres_build.exists():
shutil.rmtree(postgres_build)
build_dir = postgres_build / 'build'
if not build_dir.exists():
build_dir.mkdir(parents=True)
if run_configure or fresh_build or is_outdated:
subprocess.run([
str(postgres_src / 'configure'),
'--prefix=' + str(postgres_build / 'install'),
'--with-uuid=' + uuidlib,
], check=True, cwd=str(build_dir))
subprocess.run(
['make', 'MAKELEVEL=0', '-j', str(max(os.cpu_count() - 1, 1))],
cwd=str(build_dir), check=True)
if build_contrib or fresh_build or is_outdated:
subprocess.run(
[
'make', '-C', 'contrib', 'MAKELEVEL=0', '-j',
str(max(os.cpu_count() - 1, 1))
],
cwd=str(build_dir), check=True)
subprocess.run(
['make', 'MAKELEVEL=0', 'install'],
cwd=str(build_dir), check=True)
if build_contrib or fresh_build or is_outdated:
subprocess.run(
['make', '-C', 'contrib', 'MAKELEVEL=0', 'install'],
cwd=str(build_dir), check=True)
with open(postgres_build_stamp, 'w') as f:
f.write(source_stamp)
def _compile_postgres_extensions(build_base):
postgres_build = (build_base / 'postgres').resolve()
postgres_build_stamp_path = postgres_build / 'stamp'
ext_build = (build_base / 'ext').resolve()
ext_build_stamp_path = ext_build / 'stamp'
if postgres_build_stamp_path.exists():
with open(postgres_build_stamp_path, 'r') as f:
postgres_build_stamp = f.read()
else:
raise RuntimeError('Postgres is not built, cannot build extensions')
if ext_build_stamp_path.exists():
with open(ext_build_stamp_path, 'r') as f:
ext_build_stamp = f.read()
else:
ext_build_stamp = None
ext_dir = (pathlib.Path(__file__).parent / 'ext').resolve()
pg_config = (build_base / 'postgres' / 'install' /
'bin' / 'pg_config').resolve()
if not ext_dir.exists():
raise RuntimeError('missing Postgres extension directory')
ext_make = ['make', '-C', str(ext_dir), 'PG_CONFIG=' + str(pg_config)]
if ext_build_stamp != postgres_build_stamp:
print('Extensions build stamp does not match Postgres build stamp. '
'Rebuilding...')
subprocess.run(ext_make + ['clean'], check=True)
ext_build.mkdir(parents=True, exist_ok=True)
subprocess.run(ext_make, check=True)
subprocess.run(ext_make + ['install'], check=True)
ext_build_stamp = postgres_build_stamp
with open(ext_build_stamp_path, 'w') as f:
f.write(ext_build_stamp)
class build(distutils_build.build):
user_options = distutils_build.build.user_options + [
('pg-config=', None, 'path to pg_config to use with this build'),
('runstatedir=', None, 'directory to use for the runtime state'),
]
def initialize_options(self):
super().initialize_options()
self.pg_config = None
self.runstatedir = None
def finalize_options(self):
super().finalize_options()
def run(self, *args, **kwargs):
super().run(*args, **kwargs)
build_lib = pathlib.Path(self.build_lib)
_compile_parsers(build_lib)
if self.pg_config:
_compile_build_meta(
build_lib,
self.distribution.metadata.version,
self.pg_config,
self.runstatedir,
)
class develop(setuptools_develop.develop):
def run(self, *args, **kwargs):
_compile_parsers(pathlib.Path('build/lib'), inplace=True)
_compile_postgres(pathlib.Path('build').resolve())
_compile_postgres_extensions(pathlib.Path('build').resolve())
scripts = self.distribution.entry_points['console_scripts']
patched_scripts = [s + '_dev' for s in scripts
if not s.startswith('edgedb-server')]
patched_scripts.append('edb = edb.tools.edb:edbcommands')
self.distribution.entry_points['console_scripts'] = patched_scripts
super().run(*args, **kwargs)
class build_postgres(setuptools.Command):
description = "build postgres"
user_options = [
('configure', None, 'run ./configure'),
('build-contrib', None, 'build contrib'),
('fresh-build', None, 'rebuild from scratch'),
]
def initialize_options(self):
self.configure = False
self.build_contrib = False
self.fresh_build = False
def finalize_options(self):
pass
def run(self, *args, **kwargs):
_compile_postgres(
pathlib.Path('build').resolve(),
force_build=True,
fresh_build=self.fresh_build,
run_configure=self.configure,
build_contrib=self.build_contrib)
_compile_postgres_extensions(
pathlib.Path('build').resolve())
class build_postgres_ext(setuptools.Command):
description = "build postgres extensions"
user_options = []
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self, *args, **kwargs):
_compile_postgres_extensions(
pathlib.Path('build').resolve())
class build_ext(distutils_build_ext.build_ext):
user_options = distutils_build_ext.build_ext.user_options + [
('cython-annotate', None,
'Produce a colorized HTML version of the Cython source.'),
('cython-directives=', None,
'Cython compiler directives'),
]
def initialize_options(self):
# initialize_options() may be called multiple times on the
# same command object, so make sure not to override previously
# set options.
if getattr(self, '_initialized', False):
return
super(build_ext, self).initialize_options()
if os.environ.get('EDGEDB_DEBUG'):
self.cython_always = True
self.cython_annotate = True
self.cython_directives = "linetrace=True"
self.define = 'PG_DEBUG,CYTHON_TRACE,CYTHON_TRACE_NOGIL'
self.debug = True
else:
self.cython_always = False
self.cython_annotate = None
self.cython_directives = None
self.debug = False
def finalize_options(self):
# finalize_options() may be called multiple times on the
# same command object, so make sure not to override previously
# set options.
if getattr(self, '_initialized', False):
return
import pkg_resources
# Double check Cython presence in case setup_requires
# didn't go into effect (most likely because someone
# imported Cython before setup_requires injected the
# correct egg into sys.path.
try:
import Cython
except ImportError:
raise RuntimeError(
'please install {} to compile edgedb from source'.format(
CYTHON_DEPENDENCY))
cython_dep = pkg_resources.Requirement.parse(CYTHON_DEPENDENCY)
if Cython.__version__ not in cython_dep:
raise RuntimeError(
'edgedb requires {}, got Cython=={}'.format(
CYTHON_DEPENDENCY, Cython.__version__
))
from Cython.Build import cythonize
directives = {
'language_level': '3'
}
if self.cython_directives:
for directive in self.cython_directives.split(','):
k, _, v = directive.partition('=')
if v.lower() == 'false':
v = False
if v.lower() == 'true':
v = True
directives[k] = v
self.distribution.ext_modules[:] = cythonize(
self.distribution.ext_modules,
compiler_directives=directives,
annotate=self.cython_annotate,
include_path=["edb/server/pgproto/"])
super(build_ext, self).finalize_options()
setuptools.setup(
setup_requires=RUNTIME_DEPS + BUILD_DEPS,
use_scm_version=True,
name='edgedb-server',
description='EdgeDB Server',
author='MagicStack Inc.',
author_email='hello@magic.io',
packages=['edb'],
include_package_data=True,
cmdclass={
'build': build,
'build_ext': build_ext,
'develop': develop,
'build_postgres': build_postgres,
'build_postgres_ext': build_postgres_ext,
},
entry_points={
'console_scripts': [
'edgedb = edb.cli:cli',
'edgedb-server = edb.server.main:main',
]
},
ext_modules=[
distutils_extension.Extension(
"edb.server.pgproto.pgproto",
["edb/server/pgproto/pgproto.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.dbview.dbview",
["edb/server/dbview/dbview.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.mng_port.edgecon",
["edb/server/mng_port/edgecon.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.cache.stmt_cache",
["edb/server/cache/stmt_cache.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.pgcon.pgcon",
["edb/server/pgcon/pgcon.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.http.http",
["edb/server/http/http.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.http_edgeql_port.protocol",
["edb/server/http_edgeql_port/protocol.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.http_graphql_port.protocol",
["edb/server/http_graphql_port/protocol.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
],
install_requires=RUNTIME_DEPS,
extras_require=EXTRA_DEPS,
test_suite='tests.suite',
)
代码说明
核心代码还是setuptools.setup 中的内容，一种包含了依赖包以及扩展的配置
- 依赖
依赖包含了运行时依赖以及构建依赖
运行时依赖，主要是一些核心库，包括了pg 连接库 http 操作的，graphql 依赖库，cli解析的
RUNTIME_DEPS = [
'asyncpg~=0.18.2',
'click~=6.7',
'httptools>=0.0.13',
'immutables>=0.9',
'parsing~=1.6.1',
'prompt_toolkit>=2.0.0',
'psutil~=5.6.1',
'Pygments~=2.3.0',
'setproctitle~=1.1.10',
'setuptools_scm~=3.2.0',
'uvloop~=0.12.2',
'graphql-core~=2.1.0',
'promise~=2.2.0',
]
CYTHON_DEPENDENCY = 'Cython==0.29.6'
BUILD_DEPS = [
CYTHON_DEPENDENCY,
]
- 包名称
edgedb 基本上所有python相关的代码都在edb 中
packages=['edb'],- 自定义cmdclass 以及entry_points
详细的下边细说
# 添加了关于（pg 以及edgedb）构建，（cython，pg）扩展构建
cmdclass={
'build': build,
'build_ext': build_ext,
'develop': develop,
'build_postgres': build_postgres,
'build_postgres_ext': build_postgres_ext,
},
entry_points={
'console_scripts': [
'edgedb = edb.cli:cli',
'edgedb-server = edb.server.main:main',
]
} - 依赖的cython extension 配置
主要是cython 配置，暴露的包名以及cython 路径，以及编译选项
ext_modules=[
distutils_extension.Extension(
"edb.server.pgproto.pgproto",
["edb/server/pgproto/pgproto.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.dbview.dbview",
["edb/server/dbview/dbview.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.mng_port.edgecon",
["edb/server/mng_port/edgecon.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.cache.stmt_cache",
["edb/server/cache/stmt_cache.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.pgcon.pgcon",
["edb/server/pgcon/pgcon.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.http.http",
["edb/server/http/http.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.http_edgeql_port.protocol",
["edb/server/http_edgeql_port/protocol.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
distutils_extension.Extension(
"edb.server.http_graphql_port.protocol",
["edb/server/http_graphql_port/protocol.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
],
install_requires=RUNTIME_DEPS,
extras_require=EXTRA_DEPS,cmdclass 说明
cmdclass 是edgedb 包的核心部分，包含了cython 构建以及pg 源码构建pg extension 扩展构建
- build_ext 说明
主要是cython 相关的配置
class build_ext(distutils_build_ext.build_ext):
user_options = distutils_build_ext.build_ext.user_options + [
('cython-annotate', None,
'Produce a colorized HTML version of the Cython source.'),
('cython-directives=', None,
'Cython compiler directives'),
]
def initialize_options(self):
# initialize_options() may be called multiple times on the
# same command object, so make sure not to override previously
# set options.
if getattr(self, '_initialized', False):
return
super(build_ext, self).initialize_options()
if os.environ.get('EDGEDB_DEBUG'):
self.cython_always = True
self.cython_annotate = True
self.cython_directives = "linetrace=True"
self.define = 'PG_DEBUG,CYTHON_TRACE,CYTHON_TRACE_NOGIL'
self.debug = True
else:
self.cython_always = False
self.cython_annotate = None
self.cython_directives = None
self.debug = False
def finalize_options(self):
# finalize_options() may be called multiple times on the
# same command object, so make sure not to override previously
# set options.
if getattr(self, '_initialized', False):
return
import pkg_resources
# Double check Cython presence in case setup_requires
# didn't go into effect (most likely because someone
# imported Cython before setup_requires injected the
# correct egg into sys.path.
try:
import Cython
except ImportError:
raise RuntimeError(
'please install {} to compile edgedb from source'.format(
CYTHON_DEPENDENCY))
cython_dep = pkg_resources.Requirement.parse(CYTHON_DEPENDENCY)
if Cython.__version__ not in cython_dep:
raise RuntimeError(
'edgedb requires {}, got Cython=={}'.format(
CYTHON_DEPENDENCY, Cython.__version__
))
from Cython.Build import cythonize
directives = {
'language_level': '3'
}
if self.cython_directives:
for directive in self.cython_directives.split(','):
k, _, v = directive.partition('=')
if v.lower() == 'false':
v = False
if v.lower() == 'true':
v = True
directives[k] = v
self.distribution.ext_modules[:] = cythonize(
self.distribution.ext_modules,
compiler_directives=directives,
annotate=self.cython_annotate,
include_path=["edb/server/pgproto/"])
super(build_ext, self).finalize_options()- build_postgres pg 构建配置
pg 是edgedb 的核心，build_postgres 提供了pg 源码编译的处理
class build_postgres(setuptools.Command):
description = "build postgres"
user_options = [
('configure', None, 'run ./configure'),
('build-contrib', None, 'build contrib'),
('fresh-build', None, 'rebuild from scratch'),
]
def initialize_options(self):
self.configure = False
self.build_contrib = False
self.fresh_build = False
def finalize_options(self):
pass
def run(self, *args, **kwargs):
_compile_postgres(
pathlib.Path('build').resolve(),
force_build=True,
fresh_build=self.fresh_build,
run_configure=self.configure,
build_contrib=self.build_contrib)
_compile_postgres_extensions(
pathlib.Path('build').resolve())- build_postgres_ext 提供pg 扩展的构建处理
目前主要的扩展是提供一些工具类的封装扩展名称为 edbsys，代码在ext 中
class build_postgres_ext(setuptools.Command):
description = "build postgres extensions"
user_options = []
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self, *args, **kwargs):
_compile_postgres_extensions(
pathlib.Path('build').resolve())- build 配置
主要提供了pg_config 的配置，指定后边pg 以及pg 扩展构建的环境
class build(distutils_build.build):
user_options = distutils_build.build.user_options + [
('pg-config=', None, 'path to pg_config to use with this build'),
('runstatedir=', None, 'directory to use for the runtime state'),
]
def initialize_options(self):
super().initialize_options()
self.pg_config = None
self.runstatedir = None
def finalize_options(self):
super().finalize_options()
def run(self, *args, **kwargs):
super().run(*args, **kwargs)
build_lib = pathlib.Path(self.build_lib)
_compile_parsers(build_lib)
if self.pg_config:
_compile_build_meta(
build_lib,
self.distribution.metadata.version,
self.pg_config,
self.runstatedir,
)
说明
以上只是一个简单的分析，这个文件主要是为了进行edgedb 打包构建处理的，里面也使用了关于cython 的技术
还是值得学习的
参考资料
https://github.com/edgedb/edgedb
https://medium.com/@shamir.stav_83310/making-your-c-library-callable-from-python-by-wrapping-it-with-cython-b09db35012a3
https://github.com/stavshamir/cython-c-wrapper/
https://cython.readthedocs.io/en/latest/src/tutorial/external.html
https://cython.readthedocs.io/en/latest/src/tutorial/clibraries.html
http://pages.cs.wisc.edu/~yezheng/post/cython/
https://github.com/rongfengliang/cython-c-pip-demo/tree/local_source