Building Python C and C++ Extensions with setuptools
最后发布时间:2022-08-06 19:38:28
浏览量:
Example Python Extension Module
"""
The Pyslurm Setup - build options
"""
import os
import logging
import sys
import textwrap
import pathlib
from setuptools import setup, Extension
from distutils.dir_util import remove_tree
from distutils.version import LooseVersion
logger = logging.getLogger(__name__)
logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.DEBUG)
# Keep in sync with pyproject.toml
CYTHON_VERSION_MIN = "0.29.30"
SLURM_RELEASE = "22.5"
PYSLURM_PATCH_RELEASE = "0"
SLURM_SHARED_LIB = "libslurm.so"
CURRENT_DIR = pathlib.Path(__file__).parent
metadata = dict(
name="pyslurm",
version=SLURM_RELEASE + "." + PYSLURM_PATCH_RELEASE,
license="GPLv2",
description="Python Interface for Slurm",
long_description=(CURRENT_DIR / "README.md").read_text(),
author="Mark Roberts, Giovanni Torres, et al.",
author_email="pyslurm@googlegroups.com",
url="https://github.com/PySlurm/pyslurm",
platforms=["Linux"],
keywords=["HPC", "Batch Scheduler", "Resource Manager", "Slurm", "Cython"],
packages=["pyslurm"],
classifiers=[
"Development Status :: 5 - Production/Stable",
"Environment :: Console",
"Intended Audience :: Developers",
"Intended Audience :: System Administrators",
"License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
"Natural Language :: English",
"Operating System :: POSIX :: Linux",
"Programming Language :: Cython",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Topic :: Software Development :: Libraries",
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: System :: Distributed Computing",
],
)
class PySlurmConfig():
def __init__(self):
# Assume some defaults here
self.slurm_lib = "/usr/lib64"
self.slurm_inc = "/usr/include"
self.slurm_inc_full = "/usr/include/slurm"
config = PySlurmConfig()
def warn(log_string):
"""Warn logger"""
logger.error(log_string)
def info(log_string):
"""Info logger"""
logger.info(log_string)
if sys.version_info[:2] < (3, 6):
raise RuntimeError("Python 3.6 or higher is required to run PySlurm.")
def usage():
"""Display usage flags"""
print(
textwrap.dedent(
"""
PySlurm Help
------------
--slurm-lib=PATH Where to look for libslurm.so (default=/usr/lib64)
You can also instead use the environment
variable SLURM_LIB_DIR.
--slurm-inc=PATH Where to look for slurm.h, slurm_errno.h
and slurmdb.h (default=/usr/include)
You can also instead use the environment
variable SLURM_INCLUDE_DIR.
For help with building or installing PySlurm, please ask on the PySlurm
Google group at https://groups.google.com/forum/#!forum/pyslurm.
If you are sure that you have run into a bug, please report it at
https://github.com/PySlurm/pyslurm/issues.
"""
)
)
def inc_vers2str(hex_inc_version):
"""
Return a slurm version number string decoded from
the bit shifted components of the slurm version hex
string supplied in slurm.h
"""
a = int(hex_inc_version, 16)
b = (a >> 16 & 0xFF, a >> 8 & 0xFF, a & 0xFF)
# Only really need the major release
return f"{b[0]:02d}.{b[1]:02d}"
def read_inc_version(fname):
"""
Read the supplied include file and extract the
slurm version number in the define line e.g
#define SLURM_VERSION_NUMBER 0x020600
"""
hex_version = ""
with open(fname, "r", encoding="latin-1") as f:
for line in f:
if line.find("#define SLURM_VERSION_NUMBER") == 0:
hex_version = line.split(" ")[2].strip()
info("Detected Slurm version - "f"{inc_vers2str(hex_version)}")
if not hex_version:
raise RuntimeError("Unable to detect Slurm version")
return hex_version
def find_files_with_extension(path, extensions):
"""
Recursively find all files with specific extensions.
"""
files = [p
for p in pathlib.Path(path).glob("**/*")
if p.suffix in extensions]
return files
def cleanup_build():
"""
Cleanup build directory and temporary files
"""
info("Checking for objects to clean")
if os.path.isdir("build"):
info("Removing build/")
remove_tree("build", verbose=1)
files = find_files_with_extension("pyslurm", {".c", ".pyc"})
for file in files:
if file.is_file():
info(f"Removing: {file}")
file.unlink()
else:
raise RuntimeError(f"{file} is not a file !")
info("cleanup done")
def make_extensions():
"""
Generate Extension objects from .pyx files
"""
extensions = []
pyx_files = find_files_with_extension("pyslurm", {".pyx"})
ext_meta = {
"include_dirs": [config.slurm_inc, "."],
"library_dirs": [config.slurm_lib],
"libraries": ["slurm"],
"runtime_library_dirs": [config.slurm_lib],
}
for pyx in pyx_files:
ext = Extension(
str(pyx.with_suffix("")).replace(os.path.sep, "."),
[str(pyx)],
**ext_meta
)
extensions.append(ext)
return extensions
def parse_slurm_args():
args = sys.argv[1:]
# Check first if necessary paths to Slurm
# header and lib were provided via env var
slurm_lib = os.getenv("SLURM_LIB_DIR")
slurm_inc = os.getenv("SLURM_INCLUDE_DIR")
# If these are provided, they take precedence
# over the env vars
for arg in args:
if arg.find("--slurm-lib=") == 0:
slurm_lib = arg.split("=")[1]
sys.argv.remove(arg)
if arg.find("--slurm-inc=") == 0:
slurm_inc = arg.split("=")[1]
sys.argv.remove(arg)
if "--bgq" in args:
config.bgq = 1
if slurm_lib:
config.slurm_lib = slurm_lib
if slurm_inc:
config.slurm_inc = slurm_inc
config.slurm_inc_full = os.path.join(slurm_inc, "slurm")
def slurm_sanity_checks():
"""
Check if Slurm headers and Lib exist.
"""
if os.path.exists(f"{config.slurm_lib}/{SLURM_SHARED_LIB}"):
info(f"Found Slurm shared library in {config.slurm_lib}")
else:
raise RuntimeError(f"Cannot locate Slurm shared library in {config.slurm_lib}")
if os.path.exists(f"{config.slurm_inc_full}/slurm.h"):
info(f"Found Slurm header in {config.slurm_inc_full}")
else:
raise RuntimeError(f"Cannot locate the Slurm include in {config.slurm_inc_full}")
# Test for Slurm MAJOR.MINOR version match (ignoring .MICRO)
slurm_inc_ver = read_inc_version(f"{config.slurm_inc_full}/slurm_version.h")
major = (int(slurm_inc_ver, 16) >> 16) & 0xFF
minor = (int(slurm_inc_ver, 16) >> 8) & 0xFF
detected_version = str(major) + "." + str(minor)
if LooseVersion(detected_version) != LooseVersion(SLURM_RELEASE):
raise RuntimeError(
f"Incorrect slurm version detected, requires Slurm {SLURM_RELEASE}"
)
def cythongen():
"""
Build the PySlurm package
"""
info("Building PySlurm from source...")
try:
from Cython.Distutils import build_ext
from Cython.Build import cythonize
from Cython.Compiler.Version import version as cython_version
except ImportError as e:
msg = "Cython (https://cython.org) is required to build PySlurm."
raise RuntimeError(msg) from e
else:
if LooseVersion(cython_version) < LooseVersion(CYTHON_VERSION_MIN):
msg = f"Please use Cython version >= {CYTHON_VERSION_MIN}"
raise RuntimeError(msg)
# Clean up temporary build objects first
cleanup_build()
# Build all extensions
metadata["ext_modules"] = cythonize(make_extensions())
def parse_setuppy_commands():
"""
Parse the given setup commands
"""
args = sys.argv[1:]
if not args:
return False
# Prepend PySlurm help text when passing --help | -h
if "--help" in args or "-h" in args:
usage()
print(
textwrap.dedent(
"""
Setuptools Help
--------------
"""
)
)
return False
# Clean up all build objects
if "clean" in args:
cleanup_build()
return False
build_cmd = ('install', 'sdist', 'build', 'build_ext', 'build_py',
'build_clib', 'build_scripts', 'bdist_wheel', 'bdist_rpm',
'build_src', 'bdist_egg', 'develop')
for cmd in build_cmd:
if cmd in args:
return True
return False
def setup_package():
"""
Define the PySlurm package
"""
build_it = parse_setuppy_commands()
if build_it:
if "sdist" not in sys.argv:
parse_slurm_args()
slurm_sanity_checks()
cythongen()
setup(**metadata)
if __name__ == "__main__":
setup_package()
"""
setup.py - Setup package with the help Python's DistUtils
See https://www.python-ldap.org/ for details.
"""
import sys,os
from setuptools import setup, Extension
if sys.version_info < (3, 6):
raise RuntimeError(
'The C API from Python 3.6+ is required, found %s' % sys.version_info
)
from configparser import ConfigParser
sys.path.insert(0, os.path.join(os.getcwd(), 'Lib/ldap'))
import pkginfo
#-- A class describing the features and requirements of OpenLDAP 2.0
class OpenLDAP2:
library_dirs = []
include_dirs = []
extra_compile_args = []
extra_link_args = []
extra_objects = []
libs = ['ldap', 'lber']
defines = []
extra_files = []
LDAP_CLASS = OpenLDAP2
#-- Read the [_ldap] section of setup.cfg
cfg = ConfigParser()
cfg.read('setup.cfg')
if cfg.has_section('_ldap'):
for name in dir(LDAP_CLASS):
if cfg.has_option('_ldap', name):
setattr(LDAP_CLASS, name, cfg.get('_ldap', name).split())
for i in range(len(LDAP_CLASS.defines)):
LDAP_CLASS.defines[i]=((LDAP_CLASS.defines[i],None))
for i in range(len(LDAP_CLASS.extra_files)):
destdir, origfiles = LDAP_CLASS.extra_files[i].split(':')
origfileslist = origfiles.split(',')
LDAP_CLASS.extra_files[i]=(destdir, origfileslist)
if os.environ.get('WITH_GCOV'):
# Instrumentation for measuring code coverage
LDAP_CLASS.extra_compile_args.extend(
['-O0', '-pg', '-fprofile-arcs', '-ftest-coverage']
)
LDAP_CLASS.extra_link_args.append('-pg')
LDAP_CLASS.libs.append('gcov')
#-- Let distutils/setuptools do the rest
name = 'python-ldap'
setup(
#-- Package description
name = name,
license=pkginfo.__license__,
version=pkginfo.__version__,
description = 'Python modules for implementing LDAP clients',
long_description = """python-ldap:
python-ldap provides an object-oriented API to access LDAP directory servers
from Python programs. Mainly it wraps the OpenLDAP 2.x libs for that purpose.
Additionally the package contains modules for other LDAP-related stuff
(e.g. processing LDIF, LDAPURLs, LDAPv3 schema, LDAPv3 extended operations
and controls, etc.).
""",
author = 'python-ldap project',
author_email = 'python-ldap@python.org',
url = 'https://www.python-ldap.org/',
download_url = 'https://pypi.org/project/python-ldap/',
classifiers = [
'Development Status :: 5 - Production/Stable',
'Intended Audience :: Developers',
'Intended Audience :: System Administrators',
'Operating System :: OS Independent',
'Operating System :: MacOS :: MacOS X',
'Operating System :: Microsoft :: Windows',
'Operating System :: POSIX',
'Programming Language :: C',
'Programming Language :: Python',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
# Note: when updating Python versions, also change tox.ini and .github/workflows/*
'Topic :: Database',
'Topic :: Internet',
'Topic :: Software Development :: Libraries :: Python Modules',
'Topic :: System :: Systems Administration :: Authentication/Directory :: LDAP',
'License :: OSI Approved :: Python Software Foundation License',
],
#-- C extension modules
ext_modules = [
Extension(
'_ldap',
[
'Modules/LDAPObject.c',
'Modules/ldapcontrol.c',
'Modules/common.c',
'Modules/constants.c',
'Modules/functions.c',
'Modules/ldapmodule.c',
'Modules/message.c',
'Modules/options.c',
'Modules/berval.c',
],
depends = [
'Modules/LDAPObject.h',
'Modules/berval.h',
'Modules/common.h',
'Modules/constants_generated.h',
'Modules/constants.h',
'Modules/functions.h',
'Modules/ldapcontrol.h',
'Modules/message.h',
'Modules/options.h',
],
libraries = LDAP_CLASS.libs,
include_dirs = ['Modules'] + LDAP_CLASS.include_dirs,
library_dirs = LDAP_CLASS.library_dirs,
extra_compile_args = LDAP_CLASS.extra_compile_args,
extra_link_args = LDAP_CLASS.extra_link_args,
extra_objects = LDAP_CLASS.extra_objects,
runtime_library_dirs = (not sys.platform.startswith("win"))*LDAP_CLASS.library_dirs,
define_macros = LDAP_CLASS.defines + \
('sasl' in LDAP_CLASS.libs or 'sasl2' in LDAP_CLASS.libs or 'libsasl' in LDAP_CLASS.libs)*[('HAVE_SASL',None)] + \
('ssl' in LDAP_CLASS.libs and 'crypto' in LDAP_CLASS.libs)*[('HAVE_TLS',None)] + \
[
('LDAPMODULE_VERSION', pkginfo.__version__),
('LDAPMODULE_AUTHOR', pkginfo.__author__),
('LDAPMODULE_LICENSE', pkginfo.__license__),
]
),
],
#-- Python "stand alone" modules
py_modules = [
'ldapurl',
'ldif',
],
packages = [
'ldap',
'ldap.controls',
'ldap.extop',
'ldap.schema',
'slapdtest',
],
package_dir = {'': 'Lib',},
data_files = LDAP_CLASS.extra_files,
include_package_data=True,
install_requires=[
'pyasn1 >= 0.3.7',
'pyasn1_modules >= 0.1.5',
],
zip_safe=False,
python_requires='>=3.6',
test_suite = 'Tests',
)
Footer
running bdist_egg
running egg_info
creating demo.egg-info
writing demo.egg-info/PKG-INFO
writing dependency_links to demo.egg-info/dependency_links.txt
writing top-level names to demo.egg-info/top_level.txt
writing manifest file 'demo.egg-info/SOURCES.txt'
reading manifest file 'demo.egg-info/SOURCES.txt'
writing manifest file 'demo.egg-info/SOURCES.txt'
installing library code to build/bdist.linux-x86_64/egg
running install_lib
running build_ext
building 'demo' extension
creating build
creating build/temp.linux-x86_64-3.9
gcc -pthread -B /home/wangyang/miniconda3/compiler_compat -Wno-unused-result -Wsign-compare -DNDEBUG -O2 -Wall -fPIC -O2 -isystem /home/wangyang/miniconda3/include -I/home/wangyang/miniconda3/include -fPIC -O2 -isystem /home/wangyang/miniconda3/include -fPIC -I/home/wangyang/miniconda3/include/python3.9 -c demo.c -o build/temp.linux-x86_64-3.9/demo.o
creating build/lib.linux-x86_64-3.9
gcc -pthread -B /home/wangyang/miniconda3/compiler_compat -shared -Wl,-rpath,/home/wangyang/miniconda3/lib -Wl,-rpath-link,/home/wangyang/miniconda3/lib -L/home/wangyang/miniconda3/lib -L/home/wangyang/miniconda3/lib -Wl,-rpath,/home/wangyang/miniconda3/lib -Wl,-rpath-link,/home/wangyang/miniconda3/lib -L/home/wangyang/miniconda3/lib build/temp.linux-x86_64-3.9/demo.o -o build/lib.linux-x86_64-3.9/demo.cpython-39-x86_64-linux-gnu.so
creating build/bdist.linux-x86_64
creating build/bdist.linux-x86_64/egg
copying build/lib.linux-x86_64-3.9/demo.cpython-39-x86_64-linux-gnu.so -> build/bdist.linux-x86_64/egg
creating stub loader for demo.cpython-39-x86_64-linux-gnu.so
byte-compiling build/bdist.linux-x86_64/egg/demo.py to demo.cpython-39.pyc
creating build/bdist.linux-x86_64/egg/EGG-INFO
copying demo.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO
copying demo.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO
copying demo.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO
copying demo.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO
writing build/bdist.linux-x86_64/egg/EGG-INFO/native_libs.txt
zip_safe flag not set; analyzing archive contents...
__pycache__.demo.cpython-39: module references __file__
creating dist
creating 'dist/demo-1.0-py3.9-linux-x86_64.egg' and adding 'build/bdist.linux-x86_64/egg' to it
removing 'build/bdist.linux-x86_64/egg' (and everything under it)
Processing demo-1.0-py3.9-linux-x86_64.egg
creating /home/wangyang/miniconda3/lib/python3.9/site-packages/demo-1.0-py3.9-linux-x86_64.egg
Extracting demo-1.0-py3.9-linux-x86_64.egg to /home/wangyang/miniconda3/lib/python3.9/site-packages
Adding demo 1.0 to easy-install.pth file
Installed /home/wangyang/miniconda3/lib/python3.9/site-packages/demo-1.0-py3.9-linux-x86_64.egg
Processing dependencies for demo==1.0
Finished processing dependencies for demo==1.0
参考
