Building Python C and C++ Extensions with setuptools

最后发布时间:2022-08-06 19:38:28 浏览量:

Python、C、C扩展、Cython

Example Python Extension Module

PySlurm

"""
The Pyslurm Setup - build options
"""

import os
import logging
import sys
import textwrap
import pathlib
from setuptools import setup, Extension
from distutils.dir_util import remove_tree
from distutils.version import LooseVersion

logger = logging.getLogger(__name__)
logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.DEBUG)

# Keep in sync with pyproject.toml
CYTHON_VERSION_MIN = "0.29.30"

SLURM_RELEASE = "22.5"
PYSLURM_PATCH_RELEASE = "0"
SLURM_SHARED_LIB = "libslurm.so"
CURRENT_DIR = pathlib.Path(__file__).parent

metadata = dict(
    name="pyslurm",
    version=SLURM_RELEASE + "." + PYSLURM_PATCH_RELEASE,
    license="GPLv2",
    description="Python Interface for Slurm",
    long_description=(CURRENT_DIR / "README.md").read_text(),
    author="Mark Roberts, Giovanni Torres, et al.",
    author_email="pyslurm@googlegroups.com",
    url="https://github.com/PySlurm/pyslurm",
    platforms=["Linux"],
    keywords=["HPC", "Batch Scheduler", "Resource Manager", "Slurm", "Cython"],
    packages=["pyslurm"],
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Environment :: Console",
        "Intended Audience :: Developers",
        "Intended Audience :: System Administrators",
        "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
        "Natural Language :: English",
        "Operating System :: POSIX :: Linux",
        "Programming Language :: Cython",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "Topic :: Software Development :: Libraries",
        "Topic :: Software Development :: Libraries :: Python Modules",
        "Topic :: System :: Distributed Computing",
    ],
)

class PySlurmConfig():

    def __init__(self):
        # Assume some defaults here
        self.slurm_lib = "/usr/lib64"
        self.slurm_inc = "/usr/include"
        self.slurm_inc_full = "/usr/include/slurm"

config = PySlurmConfig()

def warn(log_string):
    """Warn logger"""
    logger.error(log_string)


def info(log_string):
    """Info logger"""
    logger.info(log_string)


if sys.version_info[:2] < (3, 6):
    raise RuntimeError("Python 3.6 or higher is required to run PySlurm.")


def usage():
    """Display usage flags"""
    print(
        textwrap.dedent(
        """
        PySlurm Help
        ------------
            --slurm-lib=PATH    Where to look for libslurm.so (default=/usr/lib64)
                                You can also instead use the environment
                                variable SLURM_LIB_DIR.
                                
            --slurm-inc=PATH    Where to look for slurm.h, slurm_errno.h
                                and slurmdb.h (default=/usr/include)
                                You can also instead use the environment
                                variable SLURM_INCLUDE_DIR.
        For help with building or installing PySlurm, please ask on the PySlurm
        Google group at https://groups.google.com/forum/#!forum/pyslurm.
        If you are sure that you have run into a bug, please report it at
        https://github.com/PySlurm/pyslurm/issues.
        """
        )
    )


def inc_vers2str(hex_inc_version):
    """
    Return a slurm version number string decoded from
    the bit shifted components of the slurm version hex
    string supplied in slurm.h
    """
    a = int(hex_inc_version, 16)
    b = (a >> 16 & 0xFF, a >> 8 & 0xFF, a & 0xFF)
    # Only really need the major release
    return f"{b[0]:02d}.{b[1]:02d}"


def read_inc_version(fname):
    """
    Read the supplied include file and extract the
    slurm version number in the define line e.g
    #define SLURM_VERSION_NUMBER 0x020600
    """
    hex_version = ""
    with open(fname, "r", encoding="latin-1") as f:
        for line in f:
            if line.find("#define SLURM_VERSION_NUMBER") == 0:
                hex_version = line.split(" ")[2].strip()
                info("Detected Slurm version - "f"{inc_vers2str(hex_version)}")

    if not hex_version:
        raise RuntimeError("Unable to detect Slurm version")

    return hex_version


def find_files_with_extension(path, extensions):
    """
    Recursively find all files with specific extensions.
    """
    files = [p
             for p in pathlib.Path(path).glob("**/*")
             if p.suffix in extensions]

    return files


def cleanup_build():
    """
    Cleanup build directory and temporary files
    """
    info("Checking for objects to clean")

    if os.path.isdir("build"):
        info("Removing build/")
        remove_tree("build", verbose=1)

    files = find_files_with_extension("pyslurm", {".c", ".pyc"})

    for file in files:
        if file.is_file():
            info(f"Removing: {file}")
            file.unlink()
        else:
            raise RuntimeError(f"{file} is not a file !")

    info("cleanup done")


def make_extensions():
    """
    Generate Extension objects from .pyx files
    """
    extensions = []
    pyx_files = find_files_with_extension("pyslurm", {".pyx"})
    ext_meta = { 
        "include_dirs": [config.slurm_inc, "."],
        "library_dirs": [config.slurm_lib],
        "libraries": ["slurm"],
        "runtime_library_dirs": [config.slurm_lib],
    }

    for pyx in pyx_files:
        ext = Extension(
                str(pyx.with_suffix("")).replace(os.path.sep, "."),
                [str(pyx)],
                **ext_meta
        )
        extensions.append(ext)

    return extensions


def parse_slurm_args():
    args = sys.argv[1:]

    # Check first if necessary paths to Slurm
    # header and lib were provided via env var
    slurm_lib = os.getenv("SLURM_LIB_DIR")
    slurm_inc = os.getenv("SLURM_INCLUDE_DIR")

    # If these are provided, they take precedence
    # over the env vars
    for arg in args:
        if arg.find("--slurm-lib=") == 0:
            slurm_lib = arg.split("=")[1]
            sys.argv.remove(arg)
        if arg.find("--slurm-inc=") == 0:
            slurm_inc = arg.split("=")[1]
            sys.argv.remove(arg)

    if "--bgq" in args:
        config.bgq = 1

    if slurm_lib:
        config.slurm_lib = slurm_lib
    if slurm_inc:
        config.slurm_inc = slurm_inc
        config.slurm_inc_full = os.path.join(slurm_inc, "slurm")


def slurm_sanity_checks():
    """
    Check if Slurm headers and Lib exist.
    """
    if os.path.exists(f"{config.slurm_lib}/{SLURM_SHARED_LIB}"):
        info(f"Found Slurm shared library in {config.slurm_lib}")
    else:
        raise RuntimeError(f"Cannot locate Slurm shared library in {config.slurm_lib}")
    
    if os.path.exists(f"{config.slurm_inc_full}/slurm.h"):
        info(f"Found Slurm header in {config.slurm_inc_full}")
    else:
        raise RuntimeError(f"Cannot locate the Slurm include in {config.slurm_inc_full}")

    # Test for Slurm MAJOR.MINOR version match (ignoring .MICRO)
    slurm_inc_ver = read_inc_version(f"{config.slurm_inc_full}/slurm_version.h")

    major = (int(slurm_inc_ver, 16) >> 16) & 0xFF
    minor = (int(slurm_inc_ver, 16) >> 8) & 0xFF
    detected_version = str(major) + "." + str(minor)

    if LooseVersion(detected_version) != LooseVersion(SLURM_RELEASE):
        raise RuntimeError(
            f"Incorrect slurm version detected, requires Slurm {SLURM_RELEASE}"
        )


def cythongen():
    """
    Build the PySlurm package
    """
    info("Building PySlurm from source...")
    try:
        from Cython.Distutils import build_ext
        from Cython.Build import cythonize
        from Cython.Compiler.Version import version as cython_version
    except ImportError as e:
        msg = "Cython (https://cython.org) is required to build PySlurm."
        raise RuntimeError(msg) from e
    else:    
        if LooseVersion(cython_version) < LooseVersion(CYTHON_VERSION_MIN):
            msg = f"Please use Cython version >= {CYTHON_VERSION_MIN}"
            raise RuntimeError(msg)


    # Clean up temporary build objects first
    cleanup_build()

    # Build all extensions
    metadata["ext_modules"] = cythonize(make_extensions()) 
    

def parse_setuppy_commands():
    """
    Parse the given setup commands
    """
    args = sys.argv[1:]

    if not args:
        return False

    # Prepend PySlurm help text when passing --help | -h
    if "--help" in args or "-h" in args:
        usage()
        print(
            textwrap.dedent(
            """
            Setuptools Help
            --------------
            """
            )
        )
        return False

    # Clean up all build objects
    if "clean" in args:
        cleanup_build()
        return False

    build_cmd = ('install', 'sdist', 'build', 'build_ext', 'build_py',
                 'build_clib', 'build_scripts', 'bdist_wheel', 'bdist_rpm',
                 'build_src', 'bdist_egg', 'develop')

    for cmd in build_cmd:
        if cmd in args:
            return True

    return False


def setup_package():
    """
    Define the PySlurm package
    """
    build_it = parse_setuppy_commands()

    if build_it:
        if "sdist" not in sys.argv:
            parse_slurm_args()
            slurm_sanity_checks()
            cythongen()

    setup(**metadata)


if __name__ == "__main__":
    setup_package()

python-ldap

"""
setup.py - Setup package with the help Python's DistUtils
See https://www.python-ldap.org/ for details.
"""

import sys,os
from setuptools import setup, Extension

if sys.version_info < (3, 6):
  raise RuntimeError(
    'The C API from Python 3.6+ is required, found %s' % sys.version_info
  )

from configparser import ConfigParser

sys.path.insert(0, os.path.join(os.getcwd(), 'Lib/ldap'))
import pkginfo

#-- A class describing the features and requirements of OpenLDAP 2.0
class OpenLDAP2:
  library_dirs = []
  include_dirs = []
  extra_compile_args = []
  extra_link_args = []
  extra_objects = []
  libs = ['ldap', 'lber']
  defines = []
  extra_files = []

LDAP_CLASS = OpenLDAP2

#-- Read the [_ldap] section of setup.cfg
cfg = ConfigParser()
cfg.read('setup.cfg')
if cfg.has_section('_ldap'):
  for name in dir(LDAP_CLASS):
    if cfg.has_option('_ldap', name):
      setattr(LDAP_CLASS, name, cfg.get('_ldap', name).split())

for i in range(len(LDAP_CLASS.defines)):
  LDAP_CLASS.defines[i]=((LDAP_CLASS.defines[i],None))

for i in range(len(LDAP_CLASS.extra_files)):
  destdir, origfiles = LDAP_CLASS.extra_files[i].split(':')
  origfileslist = origfiles.split(',')
  LDAP_CLASS.extra_files[i]=(destdir, origfileslist)

if os.environ.get('WITH_GCOV'):
  # Instrumentation for measuring code coverage
  LDAP_CLASS.extra_compile_args.extend(
    ['-O0', '-pg', '-fprofile-arcs', '-ftest-coverage']
  )
  LDAP_CLASS.extra_link_args.append('-pg')
  LDAP_CLASS.libs.append('gcov')

#-- Let distutils/setuptools do the rest
name = 'python-ldap'

setup(
  #-- Package description
  name = name,
  license=pkginfo.__license__,
  version=pkginfo.__version__,
  description = 'Python modules for implementing LDAP clients',
  long_description = """python-ldap:
  python-ldap provides an object-oriented API to access LDAP directory servers
  from Python programs. Mainly it wraps the OpenLDAP 2.x libs for that purpose.
  Additionally the package contains modules for other LDAP-related stuff
  (e.g. processing LDIF, LDAPURLs, LDAPv3 schema, LDAPv3 extended operations
  and controls, etc.).
  """,
  author = 'python-ldap project',
  author_email = 'python-ldap@python.org',
  url = 'https://www.python-ldap.org/',
  download_url = 'https://pypi.org/project/python-ldap/',
  classifiers = [
    'Development Status :: 5 - Production/Stable',
    'Intended Audience :: Developers',
    'Intended Audience :: System Administrators',
    'Operating System :: OS Independent',
    'Operating System :: MacOS :: MacOS X',
    'Operating System :: Microsoft :: Windows',
    'Operating System :: POSIX',
    'Programming Language :: C',

    'Programming Language :: Python',
    'Programming Language :: Python :: 3',
    'Programming Language :: Python :: 3.6',
    'Programming Language :: Python :: 3.7',
    'Programming Language :: Python :: 3.8',
    'Programming Language :: Python :: 3.9',
    # Note: when updating Python versions, also change tox.ini and .github/workflows/*

    'Topic :: Database',
    'Topic :: Internet',
    'Topic :: Software Development :: Libraries :: Python Modules',
    'Topic :: System :: Systems Administration :: Authentication/Directory :: LDAP',
    'License :: OSI Approved :: Python Software Foundation License',
  ],
  #-- C extension modules
  ext_modules = [
    Extension(
      '_ldap',
      [
        'Modules/LDAPObject.c',
        'Modules/ldapcontrol.c',
        'Modules/common.c',
        'Modules/constants.c',
        'Modules/functions.c',
        'Modules/ldapmodule.c',
        'Modules/message.c',
        'Modules/options.c',
        'Modules/berval.c',
      ],
      depends = [
        'Modules/LDAPObject.h',
        'Modules/berval.h',
        'Modules/common.h',
        'Modules/constants_generated.h',
        'Modules/constants.h',
        'Modules/functions.h',
        'Modules/ldapcontrol.h',
        'Modules/message.h',
        'Modules/options.h',
      ],
      libraries = LDAP_CLASS.libs,
      include_dirs = ['Modules'] + LDAP_CLASS.include_dirs,
      library_dirs = LDAP_CLASS.library_dirs,
      extra_compile_args = LDAP_CLASS.extra_compile_args,
      extra_link_args = LDAP_CLASS.extra_link_args,
      extra_objects = LDAP_CLASS.extra_objects,
      runtime_library_dirs = (not sys.platform.startswith("win"))*LDAP_CLASS.library_dirs,
      define_macros = LDAP_CLASS.defines + \
        ('sasl' in LDAP_CLASS.libs or 'sasl2' in LDAP_CLASS.libs or 'libsasl' in LDAP_CLASS.libs)*[('HAVE_SASL',None)] + \
        ('ssl' in LDAP_CLASS.libs and 'crypto' in LDAP_CLASS.libs)*[('HAVE_TLS',None)] + \
        [
          ('LDAPMODULE_VERSION', pkginfo.__version__),
          ('LDAPMODULE_AUTHOR', pkginfo.__author__),
          ('LDAPMODULE_LICENSE', pkginfo.__license__),
        ]
    ),
  ],
  #-- Python "stand alone" modules
  py_modules = [
    'ldapurl',
    'ldif',

  ],
  packages = [
    'ldap',
    'ldap.controls',
    'ldap.extop',
    'ldap.schema',
    'slapdtest',
  ],
  package_dir = {'': 'Lib',},
  data_files = LDAP_CLASS.extra_files,
  include_package_data=True,
  install_requires=[
    'pyasn1 >= 0.3.7',
    'pyasn1_modules >= 0.1.5',
  ],
  zip_safe=False,
  python_requires='>=3.6',
  test_suite = 'Tests',
)
Footer

running bdist_egg
running egg_info
creating demo.egg-info
writing demo.egg-info/PKG-INFO
writing dependency_links to demo.egg-info/dependency_links.txt
writing top-level names to demo.egg-info/top_level.txt
writing manifest file 'demo.egg-info/SOURCES.txt'
reading manifest file 'demo.egg-info/SOURCES.txt'
writing manifest file 'demo.egg-info/SOURCES.txt'
installing library code to build/bdist.linux-x86_64/egg
running install_lib
running build_ext
building 'demo' extension
creating build
creating build/temp.linux-x86_64-3.9
gcc -pthread -B /home/wangyang/miniconda3/compiler_compat -Wno-unused-result -Wsign-compare -DNDEBUG -O2 -Wall -fPIC -O2 -isystem /home/wangyang/miniconda3/include -I/home/wangyang/miniconda3/include -fPIC -O2 -isystem /home/wangyang/miniconda3/include -fPIC -I/home/wangyang/miniconda3/include/python3.9 -c demo.c -o build/temp.linux-x86_64-3.9/demo.o
creating build/lib.linux-x86_64-3.9
gcc -pthread -B /home/wangyang/miniconda3/compiler_compat -shared -Wl,-rpath,/home/wangyang/miniconda3/lib -Wl,-rpath-link,/home/wangyang/miniconda3/lib -L/home/wangyang/miniconda3/lib -L/home/wangyang/miniconda3/lib -Wl,-rpath,/home/wangyang/miniconda3/lib -Wl,-rpath-link,/home/wangyang/miniconda3/lib -L/home/wangyang/miniconda3/lib build/temp.linux-x86_64-3.9/demo.o -o build/lib.linux-x86_64-3.9/demo.cpython-39-x86_64-linux-gnu.so
creating build/bdist.linux-x86_64
creating build/bdist.linux-x86_64/egg
copying build/lib.linux-x86_64-3.9/demo.cpython-39-x86_64-linux-gnu.so -> build/bdist.linux-x86_64/egg
creating stub loader for demo.cpython-39-x86_64-linux-gnu.so
byte-compiling build/bdist.linux-x86_64/egg/demo.py to demo.cpython-39.pyc
creating build/bdist.linux-x86_64/egg/EGG-INFO
copying demo.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO
copying demo.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO
copying demo.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO
copying demo.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO
writing build/bdist.linux-x86_64/egg/EGG-INFO/native_libs.txt
zip_safe flag not set; analyzing archive contents...
__pycache__.demo.cpython-39: module references __file__
creating dist
creating 'dist/demo-1.0-py3.9-linux-x86_64.egg' and adding 'build/bdist.linux-x86_64/egg' to it
removing 'build/bdist.linux-x86_64/egg' (and everything under it)
Processing demo-1.0-py3.9-linux-x86_64.egg
creating /home/wangyang/miniconda3/lib/python3.9/site-packages/demo-1.0-py3.9-linux-x86_64.egg
Extracting demo-1.0-py3.9-linux-x86_64.egg to /home/wangyang/miniconda3/lib/python3.9/site-packages
Adding demo 1.0 to easy-install.pth file

Installed /home/wangyang/miniconda3/lib/python3.9/site-packages/demo-1.0-py3.9-linux-x86_64.egg
Processing dependencies for demo==1.0
Finished processing dependencies for demo==1.0

参考

https://elmjag.github.io/setuptools.html

快捷入口
Python 思维导图 浏览PDF 下载PDF
分享到:
标签