edgedb 内部pg 数据存储的探索 (三) 源码包setup.py 文件

edgedb 是基于python开发的,同时集成了cython 以下为包的setup.py 配置,从里面我们可以看到关于edgedb 的一些依赖
以及构建过程

setup.py 源码

整体配置不算很多,500 多行,主要是cython extension 配置以及pg 构建配置,以及pg extension 配置,其中添加了关于pg 以及
pg 扩展build 的自定义cmdclass

  • 代码
#
# This source file is part of the EdgeDB open source project.
#
# Copyright 2008-present MagicStack Inc. and the EdgeDB authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# import os.path
import pathlib
import platform
import shutil
import subprocess
import textwrap from distutils import extension as distutils_extension
from distutils.command import build as distutils_build
from distutils.command import build_ext as distutils_build_ext import setuptools
from setuptools.command import develop as setuptools_develop RUNTIME_DEPS = [
'asyncpg~=0.18.2',
'click~=6.7',
'httptools>=0.0.13',
'immutables>=0.9',
'parsing~=1.6.1',
'prompt_toolkit>=2.0.0',
'psutil~=5.6.1',
'Pygments~=2.3.0',
'setproctitle~=1.1.10',
'setuptools_scm~=3.2.0',
'uvloop~=0.12.2', 'graphql-core~=2.1.0',
'promise~=2.2.0',
] CYTHON_DEPENDENCY = 'Cython==0.29.6' DOCS_DEPS = [
'Sphinx~=2.0.0',
'lxml~=4.2.5',
'sphinxcontrib-asyncio~=0.2.0',
] BUILD_DEPS = [
CYTHON_DEPENDENCY,
] EXTRA_DEPS = {
'test': [
'flake8~=3.7.5',
'pycodestyle~=2.5.0',
'coverage~=4.5.2',
'requests-xml~=0.2.3',
'lxml',
'requests-xml',
] + DOCS_DEPS, 'docs': DOCS_DEPS,
} EXT_CFLAGS = ['-O2']
EXT_LDFLAGS = [] if platform.uname().system != 'Windows':
EXT_CFLAGS.extend([
'-std=c99', '-fsigned-char', '-Wall', '-Wsign-compare', '-Wconversion'
]) def _compile_parsers(build_lib, inplace=False):
import parsing import edb.edgeql.parser.grammar.single as edgeql_spec
import edb.edgeql.parser.grammar.block as edgeql_spec2
import edb.edgeql.parser.grammar.sdldocument as schema_spec base_path = pathlib.Path(__file__).parent.resolve() for spec in (edgeql_spec, edgeql_spec2, schema_spec):
spec_path = pathlib.Path(spec.__file__).parent
subpath = pathlib.Path(str(spec_path)[len(str(base_path)) + 1:])
pickle_name = spec.__name__.rpartition('.')[2] + '.pickle'
pickle_path = subpath / pickle_name
cache = build_lib / pickle_path
cache.parent.mkdir(parents=True, exist_ok=True)
parsing.Spec(spec, pickleFile=str(cache), verbose=True)
if inplace:
shutil.copy2(cache, base_path / pickle_path) def _compile_build_meta(build_lib, version, pg_config, runstatedir):
import pkg_resources
from edb.server import buildmeta parsed_version = buildmeta.parse_version(
pkg_resources.parse_version(version)) vertuple = list(parsed_version._asdict().values())
vertuple[2] = int(vertuple[2])
vertuple = tuple(vertuple) content = textwrap.dedent('''\
#
# This source file is part of the EdgeDB open source project.
#
# Copyright 2008-present MagicStack Inc. and the EdgeDB authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
#
# THIS FILE HAS BEEN AUTOMATICALLY GENERATED.
# PG_CONFIG_PATH = {pg_config!r}
RUNSTATE_DIR = {runstatedir!r}
VERSION = {version!r}
''').format(version=vertuple, pg_config=pg_config, runstatedir=runstatedir) directory = build_lib / 'edb' / 'server'
if not directory.exists():
directory.mkdir(parents=True) with open(directory / '_buildmeta.py', 'w+t') as f:
f.write(content) def _compile_postgres(build_base, *,
force_build=False, fresh_build=True,
run_configure=True, build_contrib=True): proc = subprocess.run(
['git', 'submodule', 'status', 'postgres'],
stdout=subprocess.PIPE, universal_newlines=True, check=True)
status = proc.stdout
if status[0] == '-':
print(
'postgres submodule not initialized, '
'run `git submodule init; git submodule update`')
exit(1) proc = subprocess.run(
['git', 'submodule', 'status', 'postgres'],
stdout=subprocess.PIPE, universal_newlines=True, check=True)
revision, _, _ = proc.stdout[1:].partition(' ')
source_stamp = proc.stdout[0] + revision postgres_build = (build_base / 'postgres').resolve()
postgres_src = (pathlib.Path(__file__).parent / 'postgres').resolve()
postgres_build_stamp = postgres_build / 'stamp' if postgres_build_stamp.exists():
with open(postgres_build_stamp, 'r') as f:
build_stamp = f.read()
else:
build_stamp = None is_outdated = source_stamp != build_stamp if is_outdated or force_build:
system = platform.system()
if system == 'Darwin':
uuidlib = 'e2fs'
elif system == 'Linux':
uuidlib = 'e2fs'
else:
raise NotImplementedError('unsupported system: {}'.format(system)) if fresh_build and postgres_build.exists():
shutil.rmtree(postgres_build)
build_dir = postgres_build / 'build'
if not build_dir.exists():
build_dir.mkdir(parents=True) if run_configure or fresh_build or is_outdated:
subprocess.run([
str(postgres_src / 'configure'),
'--prefix=' + str(postgres_build / 'install'),
'--with-uuid=' + uuidlib,
], check=True, cwd=str(build_dir)) subprocess.run(
['make', 'MAKELEVEL=0', '-j', str(max(os.cpu_count() - 1, 1))],
cwd=str(build_dir), check=True) if build_contrib or fresh_build or is_outdated:
subprocess.run(
[
'make', '-C', 'contrib', 'MAKELEVEL=0', '-j',
str(max(os.cpu_count() - 1, 1))
],
cwd=str(build_dir), check=True) subprocess.run(
['make', 'MAKELEVEL=0', 'install'],
cwd=str(build_dir), check=True) if build_contrib or fresh_build or is_outdated:
subprocess.run(
['make', '-C', 'contrib', 'MAKELEVEL=0', 'install'],
cwd=str(build_dir), check=True) with open(postgres_build_stamp, 'w') as f:
f.write(source_stamp) def _compile_postgres_extensions(build_base):
postgres_build = (build_base / 'postgres').resolve()
postgres_build_stamp_path = postgres_build / 'stamp' ext_build = (build_base / 'ext').resolve()
ext_build_stamp_path = ext_build / 'stamp' if postgres_build_stamp_path.exists():
with open(postgres_build_stamp_path, 'r') as f:
postgres_build_stamp = f.read()
else:
raise RuntimeError('Postgres is not built, cannot build extensions') if ext_build_stamp_path.exists():
with open(ext_build_stamp_path, 'r') as f:
ext_build_stamp = f.read()
else:
ext_build_stamp = None ext_dir = (pathlib.Path(__file__).parent / 'ext').resolve()
pg_config = (build_base / 'postgres' / 'install' /
'bin' / 'pg_config').resolve() if not ext_dir.exists():
raise RuntimeError('missing Postgres extension directory') ext_make = ['make', '-C', str(ext_dir), 'PG_CONFIG=' + str(pg_config)] if ext_build_stamp != postgres_build_stamp:
print('Extensions build stamp does not match Postgres build stamp. '
'Rebuilding...')
subprocess.run(ext_make + ['clean'], check=True) ext_build.mkdir(parents=True, exist_ok=True) subprocess.run(ext_make, check=True)
subprocess.run(ext_make + ['install'], check=True) ext_build_stamp = postgres_build_stamp with open(ext_build_stamp_path, 'w') as f:
f.write(ext_build_stamp) class build(distutils_build.build): user_options = distutils_build.build.user_options + [
('pg-config=', None, 'path to pg_config to use with this build'),
('runstatedir=', None, 'directory to use for the runtime state'),
] def initialize_options(self):
super().initialize_options()
self.pg_config = None
self.runstatedir = None def finalize_options(self):
super().finalize_options() def run(self, *args, **kwargs):
super().run(*args, **kwargs)
build_lib = pathlib.Path(self.build_lib)
_compile_parsers(build_lib)
if self.pg_config:
_compile_build_meta(
build_lib,
self.distribution.metadata.version,
self.pg_config,
self.runstatedir,
) class develop(setuptools_develop.develop): def run(self, *args, **kwargs):
_compile_parsers(pathlib.Path('build/lib'), inplace=True)
_compile_postgres(pathlib.Path('build').resolve())
_compile_postgres_extensions(pathlib.Path('build').resolve()) scripts = self.distribution.entry_points['console_scripts']
patched_scripts = [s + '_dev' for s in scripts
if not s.startswith('edgedb-server')]
patched_scripts.append('edb = edb.tools.edb:edbcommands')
self.distribution.entry_points['console_scripts'] = patched_scripts super().run(*args, **kwargs) class build_postgres(setuptools.Command): description = "build postgres" user_options = [
('configure', None, 'run ./configure'),
('build-contrib', None, 'build contrib'),
('fresh-build', None, 'rebuild from scratch'),
] def initialize_options(self):
self.configure = False
self.build_contrib = False
self.fresh_build = False def finalize_options(self):
pass def run(self, *args, **kwargs):
_compile_postgres(
pathlib.Path('build').resolve(),
force_build=True,
fresh_build=self.fresh_build,
run_configure=self.configure,
build_contrib=self.build_contrib) _compile_postgres_extensions(
pathlib.Path('build').resolve()) class build_postgres_ext(setuptools.Command): description = "build postgres extensions" user_options = [] def initialize_options(self):
pass def finalize_options(self):
pass def run(self, *args, **kwargs):
_compile_postgres_extensions(
pathlib.Path('build').resolve()) class build_ext(distutils_build_ext.build_ext): user_options = distutils_build_ext.build_ext.user_options + [
('cython-annotate', None,
'Produce a colorized HTML version of the Cython source.'),
('cython-directives=', None,
'Cython compiler directives'),
] def initialize_options(self):
# initialize_options() may be called multiple times on the
# same command object, so make sure not to override previously
# set options.
if getattr(self, '_initialized', False):
return super(build_ext, self).initialize_options() if os.environ.get('EDGEDB_DEBUG'):
self.cython_always = True
self.cython_annotate = True
self.cython_directives = "linetrace=True"
self.define = 'PG_DEBUG,CYTHON_TRACE,CYTHON_TRACE_NOGIL'
self.debug = True
else:
self.cython_always = False
self.cython_annotate = None
self.cython_directives = None
self.debug = False def finalize_options(self):
# finalize_options() may be called multiple times on the
# same command object, so make sure not to override previously
# set options.
if getattr(self, '_initialized', False):
return import pkg_resources # Double check Cython presence in case setup_requires
# didn't go into effect (most likely because someone
# imported Cython before setup_requires injected the
# correct egg into sys.path.
try:
import Cython
except ImportError:
raise RuntimeError(
'please install {} to compile edgedb from source'.format(
CYTHON_DEPENDENCY)) cython_dep = pkg_resources.Requirement.parse(CYTHON_DEPENDENCY)
if Cython.__version__ not in cython_dep:
raise RuntimeError(
'edgedb requires {}, got Cython=={}'.format(
CYTHON_DEPENDENCY, Cython.__version__
)) from Cython.Build import cythonize directives = {
'language_level': '3'
} if self.cython_directives:
for directive in self.cython_directives.split(','):
k, _, v = directive.partition('=')
if v.lower() == 'false':
v = False
if v.lower() == 'true':
v = True directives[k] = v self.distribution.ext_modules[:] = cythonize(
self.distribution.ext_modules,
compiler_directives=directives,
annotate=self.cython_annotate,
include_path=["edb/server/pgproto/"]) super(build_ext, self).finalize_options() setuptools.setup(
setup_requires=RUNTIME_DEPS + BUILD_DEPS,
use_scm_version=True,
name='edgedb-server',
description='EdgeDB Server',
author='MagicStack Inc.',
author_email='hello@magic.io',
packages=['edb'],
include_package_data=True,
cmdclass={
'build': build,
'build_ext': build_ext,
'develop': develop,
'build_postgres': build_postgres,
'build_postgres_ext': build_postgres_ext,
},
entry_points={
'console_scripts': [
'edgedb = edb.cli:cli',
'edgedb-server = edb.server.main:main',
]
},
ext_modules=[
distutils_extension.Extension(
"edb.server.pgproto.pgproto",
["edb/server/pgproto/pgproto.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.dbview.dbview",
["edb/server/dbview/dbview.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.mng_port.edgecon",
["edb/server/mng_port/edgecon.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.cache.stmt_cache",
["edb/server/cache/stmt_cache.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.pgcon.pgcon",
["edb/server/pgcon/pgcon.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.http.http",
["edb/server/http/http.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.http_edgeql_port.protocol",
["edb/server/http_edgeql_port/protocol.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.http_graphql_port.protocol",
["edb/server/http_graphql_port/protocol.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
],
install_requires=RUNTIME_DEPS,
extras_require=EXTRA_DEPS,
test_suite='tests.suite',
)

代码说明

核心代码还是setuptools.setup 中的内容,一种包含了依赖包以及扩展的配置

  • 依赖

    依赖包含了运行时依赖以及构建依赖

运行时依赖,主要是一些核心库,包括了pg 连接库 http 操作的,graphql 依赖库,cli解析的
RUNTIME_DEPS = [
'asyncpg~=0.18.2',
'click~=6.7',
'httptools>=0.0.13',
'immutables>=0.9',
'parsing~=1.6.1',
'prompt_toolkit>=2.0.0',
'psutil~=5.6.1',
'Pygments~=2.3.0',
'setproctitle~=1.1.10',
'setuptools_scm~=3.2.0',
'uvloop~=0.12.2',
'graphql-core~=2.1.0',
'promise~=2.2.0',
]
CYTHON_DEPENDENCY = 'Cython==0.29.6'
BUILD_DEPS = [
CYTHON_DEPENDENCY,
]
  • 包名称

    edgedb 基本上所有python相关的代码都在edb 中

    packages=['edb'],
  • 自定义cmdclass 以及entry_points

    详细的下边细说

# 添加了关于(pg 以及edgedb)构建,(cython,pg)扩展构建
cmdclass={
'build': build,
'build_ext': build_ext,
'develop': develop,
'build_postgres': build_postgres,
'build_postgres_ext': build_postgres_ext,
},
entry_points={
'console_scripts': [
'edgedb = edb.cli:cli',
'edgedb-server = edb.server.main:main',
]
}
  • 依赖的cython extension 配置

    主要是cython 配置,暴露的包名以及cython 路径,以及编译选项

ext_modules=[
distutils_extension.Extension(
"edb.server.pgproto.pgproto",
["edb/server/pgproto/pgproto.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.dbview.dbview",
["edb/server/dbview/dbview.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.mng_port.edgecon",
["edb/server/mng_port/edgecon.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.cache.stmt_cache",
["edb/server/cache/stmt_cache.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.pgcon.pgcon",
["edb/server/pgcon/pgcon.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.http.http",
["edb/server/http/http.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.http_edgeql_port.protocol",
["edb/server/http_edgeql_port/protocol.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS), distutils_extension.Extension(
"edb.server.http_graphql_port.protocol",
["edb/server/http_graphql_port/protocol.pyx"],
extra_compile_args=EXT_CFLAGS,
extra_link_args=EXT_LDFLAGS),
],
install_requires=RUNTIME_DEPS,
extras_require=EXTRA_DEPS,

cmdclass 说明

cmdclass 是edgedb 包的核心部分,包含了cython 构建以及pg 源码构建pg extension 扩展构建

  • build_ext 说明

    主要是cython 相关的配置

class build_ext(distutils_build_ext.build_ext):

    user_options = distutils_build_ext.build_ext.user_options + [
('cython-annotate', None,
'Produce a colorized HTML version of the Cython source.'),
('cython-directives=', None,
'Cython compiler directives'),
] def initialize_options(self):
# initialize_options() may be called multiple times on the
# same command object, so make sure not to override previously
# set options.
if getattr(self, '_initialized', False):
return super(build_ext, self).initialize_options() if os.environ.get('EDGEDB_DEBUG'):
self.cython_always = True
self.cython_annotate = True
self.cython_directives = "linetrace=True"
self.define = 'PG_DEBUG,CYTHON_TRACE,CYTHON_TRACE_NOGIL'
self.debug = True
else:
self.cython_always = False
self.cython_annotate = None
self.cython_directives = None
self.debug = False def finalize_options(self):
# finalize_options() may be called multiple times on the
# same command object, so make sure not to override previously
# set options.
if getattr(self, '_initialized', False):
return import pkg_resources # Double check Cython presence in case setup_requires
# didn't go into effect (most likely because someone
# imported Cython before setup_requires injected the
# correct egg into sys.path.
try:
import Cython
except ImportError:
raise RuntimeError(
'please install {} to compile edgedb from source'.format(
CYTHON_DEPENDENCY)) cython_dep = pkg_resources.Requirement.parse(CYTHON_DEPENDENCY)
if Cython.__version__ not in cython_dep:
raise RuntimeError(
'edgedb requires {}, got Cython=={}'.format(
CYTHON_DEPENDENCY, Cython.__version__
)) from Cython.Build import cythonize directives = {
'language_level': '3'
} if self.cython_directives:
for directive in self.cython_directives.split(','):
k, _, v = directive.partition('=')
if v.lower() == 'false':
v = False
if v.lower() == 'true':
v = True directives[k] = v self.distribution.ext_modules[:] = cythonize(
self.distribution.ext_modules,
compiler_directives=directives,
annotate=self.cython_annotate,
include_path=["edb/server/pgproto/"]) super(build_ext, self).finalize_options()
  • build_postgres pg 构建配置

    pg 是edgedb 的核心,build_postgres 提供了pg 源码编译的处理

class build_postgres(setuptools.Command):

    description = "build postgres"

    user_options = [
('configure', None, 'run ./configure'),
('build-contrib', None, 'build contrib'),
('fresh-build', None, 'rebuild from scratch'),
] def initialize_options(self):
self.configure = False
self.build_contrib = False
self.fresh_build = False def finalize_options(self):
pass def run(self, *args, **kwargs):
_compile_postgres(
pathlib.Path('build').resolve(),
force_build=True,
fresh_build=self.fresh_build,
run_configure=self.configure,
build_contrib=self.build_contrib) _compile_postgres_extensions(
pathlib.Path('build').resolve())
  • build_postgres_ext 提供pg 扩展的构建处理

    目前主要的扩展是提供一些工具类的封装扩展名称为 edbsys,代码在ext 中

class build_postgres_ext(setuptools.Command):
description = "build postgres extensions"
user_options = []
def initialize_options(self):
pass
def finalize_options(self):
pass
def run(self, *args, **kwargs):
_compile_postgres_extensions(
pathlib.Path('build').resolve())
  • build 配置

    主要提供了pg_config 的配置,指定后边pg 以及pg 扩展构建的环境

class build(distutils_build.build):

    user_options = distutils_build.build.user_options + [
('pg-config=', None, 'path to pg_config to use with this build'),
('runstatedir=', None, 'directory to use for the runtime state'),
] def initialize_options(self):
super().initialize_options()
self.pg_config = None
self.runstatedir = None def finalize_options(self):
super().finalize_options() def run(self, *args, **kwargs):
super().run(*args, **kwargs)
build_lib = pathlib.Path(self.build_lib)
_compile_parsers(build_lib)
if self.pg_config:
_compile_build_meta(
build_lib,
self.distribution.metadata.version,
self.pg_config,
self.runstatedir,
)

说明

以上只是一个简单的分析,这个文件主要是为了进行edgedb 打包构建处理的,里面也使用了关于cython 的技术
还是值得学习的

参考资料

https://github.com/edgedb/edgedb
https://medium.com/@shamir.stav_83310/making-your-c-library-callable-from-python-by-wrapping-it-with-cython-b09db35012a3
https://github.com/stavshamir/cython-c-wrapper/
https://cython.readthedocs.io/en/latest/src/tutorial/external.html
https://cython.readthedocs.io/en/latest/src/tutorial/clibraries.html
http://pages.cs.wisc.edu/~yezheng/post/cython/
https://github.com/rongfengliang/cython-c-pip-demo/tree/local_source

 
 
 
 
上一篇:Water Tree CodeForces 343D 树链剖分+线段树


下一篇:HDU 5293 Tree chain problem