Python、C、C扩展、Cython
Example Python Extension Module
PySlurm
""" The Pyslurm Setup - build options """ import os import logging import sys import textwrap import pathlib from setuptools import setup, Extension from distutils.dir_util import remove_tree from distutils.version import LooseVersion logger = logging.getLogger(__name__) logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.DEBUG) # Keep in sync with pyproject.toml CYTHON_VERSION_MIN = "0.29.30" SLURM_RELEASE = "22.5" PYSLURM_PATCH_RELEASE = "0" SLURM_SHARED_LIB = "libslurm.so" CURRENT_DIR = pathlib.Path(__file__).parent metadata = dict( name="pyslurm", version=SLURM_RELEASE + "." + PYSLURM_PATCH_RELEASE, license="GPLv2", description="Python Interface for Slurm", long_description=(CURRENT_DIR / "README.md").read_text(), author="Mark Roberts, Giovanni Torres, et al.", author_email="pyslurm@googlegroups.com", url="https://github.com/PySlurm/pyslurm", platforms=["Linux"], keywords=["HPC", "Batch Scheduler", "Resource Manager", "Slurm", "Cython"], packages=["pyslurm"], classifiers=[ "Development Status :: 5 - Production/Stable", "Environment :: Console", "Intended Audience :: Developers", "Intended Audience :: System Administrators", "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", "Natural Language :: English", "Operating System :: POSIX :: Linux", "Programming Language :: Cython", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Topic :: Software Development :: Libraries", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: System :: Distributed Computing", ], ) class PySlurmConfig(): def __init__(self): # Assume some defaults here self.slurm_lib = "/usr/lib64" self.slurm_inc = "/usr/include" self.slurm_inc_full = "/usr/include/slurm" config = PySlurmConfig() def warn(log_string): """Warn logger""" logger.error(log_string) def info(log_string): """Info logger""" logger.info(log_string) if sys.version_info[:2] < (3, 6): raise RuntimeError("Python 3.6 or higher is required to run PySlurm.") def usage(): """Display usage flags""" print( textwrap.dedent( """ PySlurm Help ------------ --slurm-lib=PATH Where to look for libslurm.so (default=/usr/lib64) You can also instead use the environment variable SLURM_LIB_DIR. --slurm-inc=PATH Where to look for slurm.h, slurm_errno.h and slurmdb.h (default=/usr/include) You can also instead use the environment variable SLURM_INCLUDE_DIR. For help with building or installing PySlurm, please ask on the PySlurm Google group at https://groups.google.com/forum/#!forum/pyslurm. If you are sure that you have run into a bug, please report it at https://github.com/PySlurm/pyslurm/issues. """ ) ) def inc_vers2str(hex_inc_version): """ Return a slurm version number string decoded from the bit shifted components of the slurm version hex string supplied in slurm.h """ a = int(hex_inc_version, 16) b = (a >> 16 & 0xFF, a >> 8 & 0xFF, a & 0xFF) # Only really need the major release return f"{b[0]:02d}.{b[1]:02d}" def read_inc_version(fname): """ Read the supplied include file and extract the slurm version number in the define line e.g #define SLURM_VERSION_NUMBER 0x020600 """ hex_version = "" with open(fname, "r", encoding="latin-1") as f: for line in f: if line.find("#define SLURM_VERSION_NUMBER") == 0: hex_version = line.split(" ")[2].strip() info("Detected Slurm version - "f"{inc_vers2str(hex_version)}") if not hex_version: raise RuntimeError("Unable to detect Slurm version") return hex_version def find_files_with_extension(path, extensions): """ Recursively find all files with specific extensions. """ files = [p for p in pathlib.Path(path).glob("**/*") if p.suffix in extensions] return files def cleanup_build(): """ Cleanup build directory and temporary files """ info("Checking for objects to clean") if os.path.isdir("build"): info("Removing build/") remove_tree("build", verbose=1) files = find_files_with_extension("pyslurm", {".c", ".pyc"}) for file in files: if file.is_file(): info(f"Removing: {file}") file.unlink() else: raise RuntimeError(f"{file} is not a file !") info("cleanup done") def make_extensions(): """ Generate Extension objects from .pyx files """ extensions = [] pyx_files = find_files_with_extension("pyslurm", {".pyx"}) ext_meta = { "include_dirs": [config.slurm_inc, "."], "library_dirs": [config.slurm_lib], "libraries": ["slurm"], "runtime_library_dirs": [config.slurm_lib], } for pyx in pyx_files: ext = Extension( str(pyx.with_suffix("")).replace(os.path.sep, "."), [str(pyx)], **ext_meta ) extensions.append(ext) return extensions def parse_slurm_args(): args = sys.argv[1:] # Check first if necessary paths to Slurm # header and lib were provided via env var slurm_lib = os.getenv("SLURM_LIB_DIR") slurm_inc = os.getenv("SLURM_INCLUDE_DIR") # If these are provided, they take precedence # over the env vars for arg in args: if arg.find("--slurm-lib=") == 0: slurm_lib = arg.split("=")[1] sys.argv.remove(arg) if arg.find("--slurm-inc=") == 0: slurm_inc = arg.split("=")[1] sys.argv.remove(arg) if "--bgq" in args: config.bgq = 1 if slurm_lib: config.slurm_lib = slurm_lib if slurm_inc: config.slurm_inc = slurm_inc config.slurm_inc_full = os.path.join(slurm_inc, "slurm") def slurm_sanity_checks(): """ Check if Slurm headers and Lib exist. """ if os.path.exists(f"{config.slurm_lib}/{SLURM_SHARED_LIB}"): info(f"Found Slurm shared library in {config.slurm_lib}") else: raise RuntimeError(f"Cannot locate Slurm shared library in {config.slurm_lib}") if os.path.exists(f"{config.slurm_inc_full}/slurm.h"): info(f"Found Slurm header in {config.slurm_inc_full}") else: raise RuntimeError(f"Cannot locate the Slurm include in {config.slurm_inc_full}") # Test for Slurm MAJOR.MINOR version match (ignoring .MICRO) slurm_inc_ver = read_inc_version(f"{config.slurm_inc_full}/slurm_version.h") major = (int(slurm_inc_ver, 16) >> 16) & 0xFF minor = (int(slurm_inc_ver, 16) >> 8) & 0xFF detected_version = str(major) + "." + str(minor) if LooseVersion(detected_version) != LooseVersion(SLURM_RELEASE): raise RuntimeError( f"Incorrect slurm version detected, requires Slurm {SLURM_RELEASE}" ) def cythongen(): """ Build the PySlurm package """ info("Building PySlurm from source...") try: from Cython.Distutils import build_ext from Cython.Build import cythonize from Cython.Compiler.Version import version as cython_version except ImportError as e: msg = "Cython (https://cython.org) is required to build PySlurm." raise RuntimeError(msg) from e else: if LooseVersion(cython_version) < LooseVersion(CYTHON_VERSION_MIN): msg = f"Please use Cython version >= {CYTHON_VERSION_MIN}" raise RuntimeError(msg) # Clean up temporary build objects first cleanup_build() # Build all extensions metadata["ext_modules"] = cythonize(make_extensions()) def parse_setuppy_commands(): """ Parse the given setup commands """ args = sys.argv[1:] if not args: return False # Prepend PySlurm help text when passing --help | -h if "--help" in args or "-h" in args: usage() print( textwrap.dedent( """ Setuptools Help -------------- """ ) ) return False # Clean up all build objects if "clean" in args: cleanup_build() return False build_cmd = ('install', 'sdist', 'build', 'build_ext', 'build_py', 'build_clib', 'build_scripts', 'bdist_wheel', 'bdist_rpm', 'build_src', 'bdist_egg', 'develop') for cmd in build_cmd: if cmd in args: return True return False def setup_package(): """ Define the PySlurm package """ build_it = parse_setuppy_commands() if build_it: if "sdist" not in sys.argv: parse_slurm_args() slurm_sanity_checks() cythongen() setup(**metadata) if __name__ == "__main__": setup_package()
python-ldap
""" setup.py - Setup package with the help Python's DistUtils See https://www.python-ldap.org/ for details. """ import sys,os from setuptools import setup, Extension if sys.version_info < (3, 6): raise RuntimeError( 'The C API from Python 3.6+ is required, found %s' % sys.version_info ) from configparser import ConfigParser sys.path.insert(0, os.path.join(os.getcwd(), 'Lib/ldap')) import pkginfo #-- A class describing the features and requirements of OpenLDAP 2.0 class OpenLDAP2: library_dirs = [] include_dirs = [] extra_compile_args = [] extra_link_args = [] extra_objects = [] libs = ['ldap', 'lber'] defines = [] extra_files = [] LDAP_CLASS = OpenLDAP2 #-- Read the [_ldap] section of setup.cfg cfg = ConfigParser() cfg.read('setup.cfg') if cfg.has_section('_ldap'): for name in dir(LDAP_CLASS): if cfg.has_option('_ldap', name): setattr(LDAP_CLASS, name, cfg.get('_ldap', name).split()) for i in range(len(LDAP_CLASS.defines)): LDAP_CLASS.defines[i]=((LDAP_CLASS.defines[i],None)) for i in range(len(LDAP_CLASS.extra_files)): destdir, origfiles = LDAP_CLASS.extra_files[i].split(':') origfileslist = origfiles.split(',') LDAP_CLASS.extra_files[i]=(destdir, origfileslist) if os.environ.get('WITH_GCOV'): # Instrumentation for measuring code coverage LDAP_CLASS.extra_compile_args.extend( ['-O0', '-pg', '-fprofile-arcs', '-ftest-coverage'] ) LDAP_CLASS.extra_link_args.append('-pg') LDAP_CLASS.libs.append('gcov') #-- Let distutils/setuptools do the rest name = 'python-ldap' setup( #-- Package description name = name, license=pkginfo.__license__, version=pkginfo.__version__, description = 'Python modules for implementing LDAP clients', long_description = """python-ldap: python-ldap provides an object-oriented API to access LDAP directory servers from Python programs. Mainly it wraps the OpenLDAP 2.x libs for that purpose. Additionally the package contains modules for other LDAP-related stuff (e.g. processing LDIF, LDAPURLs, LDAPv3 schema, LDAPv3 extended operations and controls, etc.). """, author = 'python-ldap project', author_email = 'python-ldap@python.org', url = 'https://www.python-ldap.org/', download_url = 'https://pypi.org/project/python-ldap/', classifiers = [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Intended Audience :: System Administrators', 'Operating System :: OS Independent', 'Operating System :: MacOS :: MacOS X', 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX', 'Programming Language :: C', 'Programming Language :: Python', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', # Note: when updating Python versions, also change tox.ini and .github/workflows/* 'Topic :: Database', 'Topic :: Internet', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: System :: Systems Administration :: Authentication/Directory :: LDAP', 'License :: OSI Approved :: Python Software Foundation License', ], #-- C extension modules ext_modules = [ Extension( '_ldap', [ 'Modules/LDAPObject.c', 'Modules/ldapcontrol.c', 'Modules/common.c', 'Modules/constants.c', 'Modules/functions.c', 'Modules/ldapmodule.c', 'Modules/message.c', 'Modules/options.c', 'Modules/berval.c', ], depends = [ 'Modules/LDAPObject.h', 'Modules/berval.h', 'Modules/common.h', 'Modules/constants_generated.h', 'Modules/constants.h', 'Modules/functions.h', 'Modules/ldapcontrol.h', 'Modules/message.h', 'Modules/options.h', ], libraries = LDAP_CLASS.libs, include_dirs = ['Modules'] + LDAP_CLASS.include_dirs, library_dirs = LDAP_CLASS.library_dirs, extra_compile_args = LDAP_CLASS.extra_compile_args, extra_link_args = LDAP_CLASS.extra_link_args, extra_objects = LDAP_CLASS.extra_objects, runtime_library_dirs = (not sys.platform.startswith("win"))*LDAP_CLASS.library_dirs, define_macros = LDAP_CLASS.defines + \ ('sasl' in LDAP_CLASS.libs or 'sasl2' in LDAP_CLASS.libs or 'libsasl' in LDAP_CLASS.libs)*[('HAVE_SASL',None)] + \ ('ssl' in LDAP_CLASS.libs and 'crypto' in LDAP_CLASS.libs)*[('HAVE_TLS',None)] + \ [ ('LDAPMODULE_VERSION', pkginfo.__version__), ('LDAPMODULE_AUTHOR', pkginfo.__author__), ('LDAPMODULE_LICENSE', pkginfo.__license__), ] ), ], #-- Python "stand alone" modules py_modules = [ 'ldapurl', 'ldif', ], packages = [ 'ldap', 'ldap.controls', 'ldap.extop', 'ldap.schema', 'slapdtest', ], package_dir = {'': 'Lib',}, data_files = LDAP_CLASS.extra_files, include_package_data=True, install_requires=[ 'pyasn1 >= 0.3.7', 'pyasn1_modules >= 0.1.5', ], zip_safe=False, python_requires='>=3.6', test_suite = 'Tests', ) Footer
running bdist_egg running egg_info creating demo.egg-info writing demo.egg-info/PKG-INFO writing dependency_links to demo.egg-info/dependency_links.txt writing top-level names to demo.egg-info/top_level.txt writing manifest file 'demo.egg-info/SOURCES.txt' reading manifest file 'demo.egg-info/SOURCES.txt' writing manifest file 'demo.egg-info/SOURCES.txt' installing library code to build/bdist.linux-x86_64/egg running install_lib running build_ext building 'demo' extension creating build creating build/temp.linux-x86_64-3.9 gcc -pthread -B /home/wangyang/miniconda3/compiler_compat -Wno-unused-result -Wsign-compare -DNDEBUG -O2 -Wall -fPIC -O2 -isystem /home/wangyang/miniconda3/include -I/home/wangyang/miniconda3/include -fPIC -O2 -isystem /home/wangyang/miniconda3/include -fPIC -I/home/wangyang/miniconda3/include/python3.9 -c demo.c -o build/temp.linux-x86_64-3.9/demo.o creating build/lib.linux-x86_64-3.9 gcc -pthread -B /home/wangyang/miniconda3/compiler_compat -shared -Wl,-rpath,/home/wangyang/miniconda3/lib -Wl,-rpath-link,/home/wangyang/miniconda3/lib -L/home/wangyang/miniconda3/lib -L/home/wangyang/miniconda3/lib -Wl,-rpath,/home/wangyang/miniconda3/lib -Wl,-rpath-link,/home/wangyang/miniconda3/lib -L/home/wangyang/miniconda3/lib build/temp.linux-x86_64-3.9/demo.o -o build/lib.linux-x86_64-3.9/demo.cpython-39-x86_64-linux-gnu.so creating build/bdist.linux-x86_64 creating build/bdist.linux-x86_64/egg copying build/lib.linux-x86_64-3.9/demo.cpython-39-x86_64-linux-gnu.so -> build/bdist.linux-x86_64/egg creating stub loader for demo.cpython-39-x86_64-linux-gnu.so byte-compiling build/bdist.linux-x86_64/egg/demo.py to demo.cpython-39.pyc creating build/bdist.linux-x86_64/egg/EGG-INFO copying demo.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO copying demo.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO copying demo.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO copying demo.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO writing build/bdist.linux-x86_64/egg/EGG-INFO/native_libs.txt zip_safe flag not set; analyzing archive contents... __pycache__.demo.cpython-39: module references __file__ creating dist creating 'dist/demo-1.0-py3.9-linux-x86_64.egg' and adding 'build/bdist.linux-x86_64/egg' to it removing 'build/bdist.linux-x86_64/egg' (and everything under it) Processing demo-1.0-py3.9-linux-x86_64.egg creating /home/wangyang/miniconda3/lib/python3.9/site-packages/demo-1.0-py3.9-linux-x86_64.egg Extracting demo-1.0-py3.9-linux-x86_64.egg to /home/wangyang/miniconda3/lib/python3.9/site-packages Adding demo 1.0 to easy-install.pth file Installed /home/wangyang/miniconda3/lib/python3.9/site-packages/demo-1.0-py3.9-linux-x86_64.egg Processing dependencies for demo==1.0 Finished processing dependencies for demo==1.0
https://elmjag.github.io/setuptools.html