setup.py

#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vi:ts=4:et

"""Setup script for the PycURL module distribution."""

PACKAGE = "pycurl"
PY_PACKAGE = "curl"
VERSION = "7.19.5"

import glob, os, re, sys, string, subprocess
import distutils
from distutils.core import setup
from distutils.extension import Extension
from distutils.util import split_quoted
from distutils.version import LooseVersion

try:
 # python 2
 exception_base = StandardError
except NameError:
 # python 3
 exception_base = Exception
class ConfigurationError(exception_base):
 pass


def fail(msg):
 sys.stderr.write(msg + "\n")
 exit(10)


def scan_argv(s, default=None):
 p = default
 i = 1
 while i < len(sys.argv):
 arg = sys.argv[i]
 if str.find(arg, s) == 0:
 if s.endswith('='):
 # --option=value
 p = arg[len(s):]
 assert p, arg
 else:
 # --option
 # set value to True
 p = True
 del sys.argv[i]
 else:
 i = i + 1
 ##print sys.argv
 return p


class ExtensionConfiguration(object):
 def __init__(self):
 self.include_dirs = []
 self.define_macros = [("PYCURL_VERSION", '"%s"' % VERSION)]
 self.library_dirs = []
 self.libraries = []
 self.runtime_library_dirs = []
 self.extra_objects = []
 self.extra_compile_args = []
 self.extra_link_args = []
 
 self.configure()

 @property
 def define_symbols(self):
 return [symbol for symbol, expansion in self.define_macros]

 # append contents of an environment variable to library_dirs[]
 def add_libdirs(self, envvar, sep, fatal=False):
 v = os.environ.get(envvar)
 if not v:
 return
 for dir in str.split(v, sep):
 dir = str.strip(dir)
 if not dir:
 continue
 dir = os.path.normpath(dir)
 if os.path.isdir(dir):
 if not dir in library_dirs:
 self.library_dirs.append(dir)
 elif fatal:
 fail("FATAL: bad directory %s in environment variable %s" % (dir, envvar))


 def configure_unix(self):
 OPENSSL_DIR = scan_argv("--openssl-dir=")
 if OPENSSL_DIR is not None:
 self.include_dirs.append(os.path.join(OPENSSL_DIR, "include"))
 CURL_CONFIG = os.environ.get('PYCURL_CURL_CONFIG', "curl-config")
 CURL_CONFIG = scan_argv("--curl-config=", CURL_CONFIG)
 try:
 p = subprocess.Popen((CURL_CONFIG, '--version'),
 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 except OSError:
 exc = sys.exc_info()[1]
 msg = 'Could not run curl-config: %s' % str(exc)
 raise ConfigurationError(msg)
 stdout, stderr = p.communicate()
 if p.wait() != 0:
 msg = "`%s' not found -- please install the libcurl development files or specify --curl-config=/path/to/curl-config" % CURL_CONFIG
 if stderr:
 msg += ":\n" + stderr.decode()
 raise ConfigurationError(msg)
 libcurl_version = stdout.decode().strip()
 print("Using %s (%s)" % (CURL_CONFIG, libcurl_version))
 p = subprocess.Popen((CURL_CONFIG, '--cflags'),
 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 stdout, stderr = p.communicate()
 if p.wait() != 0:
 msg = "Problem running `%s' --cflags" % CURL_CONFIG
 if stderr:
 msg += ":\n" + stderr.decode()
 raise ConfigurationError(msg)
 for arg in split_quoted(stdout.decode()):
 if arg[:2] == "-I":
 # do not add /usr/include
 if not re.search(r"^\/+usr\/+include\/*$", arg[2:]):
 self.include_dirs.append(arg[2:])
 else:
 self.extra_compile_args.append(arg)

 # Obtain linker flags/libraries to link against.
 # In theory, all we should need is `curl-config --libs`.
 # Apparently on some platforms --libs fails and --static-libs works,
 # so try that.
 # If --libs succeeds do not try --static-libs; see
 # https://github.com/pycurl/pycurl/issues/52 for more details.
 # If neither --libs nor --static-libs work, fail.
 #
 # --libs/--static-libs are also used for SSL detection.
 # libcurl may be configured such that --libs only includes -lcurl
 # without any of libcurl's dependent libraries, but the dependent
 # libraries would be included in --static-libs (unless libcurl
 # was built with static libraries disabled).
 # Therefore we largely ignore (see below) --static-libs output for
 # libraries and flags if --libs succeeded, but consult both outputs
 # for hints as to which SSL library libcurl is linked against.
 # More information: https://github.com/pycurl/pycurl/pull/147
 #
 # The final point is we should link agaist the SSL library in use
 # even if libcurl does not tell us to, because *we* invoke functions
 # in that SSL library. This means any SSL libraries found in
 # --static-libs are forwarded to our libraries.
 optbuf = ''
 sslhintbuf = ''
 errtext = ''
 for option in ["--libs", "--static-libs"]:
 p = subprocess.Popen((CURL_CONFIG, option),
 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 stdout, stderr = p.communicate()
 if p.wait() == 0:
 if optbuf == '':
 # first successful call
 optbuf = stdout.decode()
 # optbuf only has output from this call
 sslhintbuf += optbuf
 else:
 # second successful call
 sslhintbuf += stdout.decode()
 else:
 if optbuf == '':
 # no successful call yet
 errtext += stderr.decode()
 else:
 # first call succeeded and second call failed
 # ignore stderr and the error exit
 pass
 if optbuf == "":
 msg = "Neither curl-config --libs nor curl-config --static-libs" +\
 " succeeded and produced output"
 if errtext:
 msg += ":\n" + errtext
 raise ConfigurationError(msg)
 
 ssl_lib_detected = False
 if 'PYCURL_SSL_LIBRARY' in os.environ:
 ssl_lib = os.environ['PYCURL_SSL_LIBRARY']
 if ssl_lib in ['openssl', 'gnutls', 'nss']:
 ssl_lib_detected = True
 self.define_macros.append(('HAVE_CURL_%s' % ssl_lib.upper(), 1))
 else:
 raise ConfigurationError('Invalid value "%s" for PYCURL_SSL_LIBRARY' % ssl_lib)
 ssl_options = {
 '--with-ssl': 'HAVE_CURL_OPENSSL',
 '--with-gnutls': 'HAVE_CURL_GNUTLS',
 '--with-nss': 'HAVE_CURL_NSS',
 }
 for option in ssl_options:
 if scan_argv(option) is not None:
 for other_option in ssl_options:
 if option != other_option:
 if scan_argv(other_option) is not None:
 raise ConfigurationError('Cannot give both %s and %s' % (option, other_option))
 ssl_lib_detected = True
 self.define_macros.append((ssl_options[option], 1))

 # libraries and options - all libraries and options are forwarded
 # but if --libs succeeded, --static-libs output is ignored
 for arg in split_quoted(optbuf):
 if arg[:2] == "-l":
 self.libraries.append(arg[2:])
 elif arg[:2] == "-L":
 self.library_dirs.append(arg[2:])
 else:
 self.extra_link_args.append(arg)
 # ssl detection - ssl libraries are forwarded
 for arg in split_quoted(sslhintbuf):
 if arg[:2] == "-l":
 if not ssl_lib_detected and arg[2:] == 'ssl':
 self.define_macros.append(('HAVE_CURL_OPENSSL', 1))
 ssl_lib_detected = True
 # the actual library that defines CRYPTO_num_locks etc.
 # is crypto, and on cygwin linking against ssl does not
 # link against crypto as of May 2014.
 # http://stackoverflow.com/questions/23687488/cant-get-pycurl-to-install-on-cygwin-missing-openssl-symbols-crypto-num-locks
 self.libraries.append('crypto')
 if not ssl_lib_detected and arg[2:] == 'gnutls':
 self.define_macros.append(('HAVE_CURL_GNUTLS', 1))
 ssl_lib_detected = True
 self.libraries.append('gnutls')
 if not ssl_lib_detected and arg[2:] == 'ssl3':
 self.define_macros.append(('HAVE_CURL_NSS', 1))
 ssl_lib_detected = True
 self.libraries.append('ssl3')
 if not ssl_lib_detected:
 p = subprocess.Popen((CURL_CONFIG, '--features'),
 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 stdout, stderr = p.communicate()
 if p.wait() != 0:
 msg = "Problem running `%s' --features" % CURL_CONFIG
 if stderr:
 msg += ":\n" + stderr.decode()
 raise ConfigurationError(msg)
 for feature in split_quoted(stdout.decode()):
 if feature == 'SSL':
 # this means any ssl library, not just openssl
 self.define_macros.append(('HAVE_CURL_SSL', 1))
 else:
 # if we are configuring for a particular ssl library,
 # we can assume that ssl is being used
 self.define_macros.append(('HAVE_CURL_SSL', 1))
 if not self.libraries:
 self.libraries.append("curl")
 
 # Add extra compile flag for MacOS X
 if sys.platform[:-1] == "darwin":
 self.extra_link_args.append("-flat_namespace")
 
 # Recognize --avoid-stdio on Unix so that it can be tested
 self.check_avoid_stdio()


 def configure_windows(self):
 # Windows users have to pass --curl-dir parameter to specify path
 # to libcurl, because there is no curl-config on windows at all.
 curl_dir = scan_argv("--curl-dir=")
 if curl_dir is None:
 fail("Please specify --curl-dir=/path/to/built/libcurl")
 if not os.path.exists(curl_dir):
 fail("Curl directory does not exist: %s" % curl_dir)
 if not os.path.isdir(curl_dir):
 fail("Curl directory is not a directory: %s" % curl_dir)
 print("Using curl directory: %s" % curl_dir)
 self.include_dirs.append(os.path.join(curl_dir, "include"))

 # libcurl windows documentation states that for linking against libcurl
 # dll, the import library name is libcurl_imp.lib.
 # in practice, the library name sometimes is libcurl.lib.
 # override with: --libcurl-lib-name=libcurl_imp.lib
 curl_lib_name = scan_argv('--libcurl-lib-name=', 'libcurl.lib')

 if scan_argv("--use-libcurl-dll") is not None:
 libcurl_lib_path = os.path.join(curl_dir, "lib", curl_lib_name)
 self.extra_link_args.extend(["ws2_32.lib"])
 if str.find(sys.version, "MSC") >= 0:
 # build a dll
 self.extra_compile_args.append("-MD")
 else:
 self.extra_compile_args.append("-DCURL_STATICLIB")
 libcurl_lib_path = os.path.join(curl_dir, "lib", curl_lib_name)
 self.extra_link_args.extend(["gdi32.lib", "wldap32.lib", "winmm.lib", "ws2_32.lib",])

 if not os.path.exists(libcurl_lib_path):
 fail("libcurl.lib does not exist at %s.\nCurl directory must point to compiled libcurl (bin/include/lib subdirectories): %s" %(libcurl_lib_path, curl_dir))
 self.extra_objects.append(libcurl_lib_path)
 
 self.check_avoid_stdio()
 
 # make pycurl binary work on windows xp.
 # we use inet_ntop which was added in vista and implement a fallback.
 # our implementation will not be compiled with _WIN32_WINNT targeting
 # vista or above, thus said binary won't work on xp.
 # http://curl.haxx.se/mail/curlpython-2013-12/0007.html
 self.extra_compile_args.append("-D_WIN32_WINNT=0x0501")

 if str.find(sys.version, "MSC") >= 0:
 self.extra_compile_args.append("-O2")
 self.extra_compile_args.append("-GF") # enable read-only string pooling
 self.extra_compile_args.append("-WX") # treat warnings as errors
 p = subprocess.Popen(['cl.exe'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 out, err = p.communicate()
 match = re.search(r'Version (\d+)', err.decode().split("\n")[0])
 if match and int(match.group(1)) < 16:
 # option removed in vs 2010:
 # connect.microsoft.com/VisualStudio/feedback/details/475896/link-fatal-error-lnk1117-syntax-error-in-option-opt-nowin98/
 self.extra_link_args.append("/opt:nowin98") # use small section alignment

 if sys.platform == "win32":
 configure = configure_windows
 else:
 configure = configure_unix
 
 
 def check_avoid_stdio(self):
 if 'PYCURL_SETUP_OPTIONS' in os.environ and '--avoid-stdio' in os.environ['PYCURL_SETUP_OPTIONS']:
 self.extra_compile_args.append("-DPYCURL_AVOID_STDIO")
 if scan_argv('--avoid-stdio') is not None:
 self.extra_compile_args.append("-DPYCURL_AVOID_STDIO")

def get_bdist_msi_version_hack():
 # workaround for distutils/msi version requirement per
 # epydoc.sourceforge.net/stdlib/distutils.version.StrictVersion-class.html -
 # only x.y.z version numbers are supported, whereas our versions might be x.y.z.p.
 # bugs.python.org/issue6040#msg133094
 from distutils.command.bdist_msi import bdist_msi
 import inspect
 import types
 import re
 
 class bdist_msi_version_hack(bdist_msi):
 """ MSI builder requires version to be in the x.x.x format """
 def run(self):
 def monkey_get_version(self):
 """ monkey patch replacement for metadata.get_version() that
 returns MSI compatible version string for bdist_msi
 """
 # get filename of the calling function
 if inspect.stack()[1][1].endswith('bdist_msi.py'):
 # strip revision from version (if any), e.g. 11.0.0-r31546
 match = re.match(r'(\d+\.\d+\.\d+)', self.version)
 assert match
 return match.group(1)
 else:
 return self.version

 # monkeypatching get_version() call for DistributionMetadata
 self.distribution.metadata.get_version = \
 types.MethodType(monkey_get_version, self.distribution.metadata)
 bdist_msi.run(self)
 
 return bdist_msi_version_hack


def strip_pycurl_options():
 if sys.platform == 'win32':
 options = [
 '--curl-dir=', '--curl-lib-name=', '--use-libcurl-dll',
 '--avoid-stdio',
 ]
 else:
 options = ['--openssl-dir=', '--curl-config=', '--avoid-stdio']
 for option in options:
 scan_argv(option)


###############################################################################

def get_extension(split_extension_source=False):
 if split_extension_source:
 sources = [
 os.path.join("src", "docstrings.c"),
 os.path.join("src", "easy.c"),
 os.path.join("src", "module.c"),
 os.path.join("src", "multi.c"),
 os.path.join("src", "oscompat.c"),
 os.path.join("src", "pythoncompat.c"),
 os.path.join("src", "share.c"),
 os.path.join("src", "stringcompat.c"),
 os.path.join("src", "threadsupport.c"),
 ]
 depends = [
 os.path.join("src", "pycurl.h"),
 ]
 else:
 sources = [
 os.path.join("src", "allpycurl.c"),
 ]
 depends = []
 ext_config = ExtensionConfiguration()
 ext = Extension(
 name=PACKAGE,
 sources=sources,
 depends=depends,
 include_dirs=ext_config.include_dirs,
 define_macros=ext_config.define_macros,
 library_dirs=ext_config.library_dirs,
 libraries=ext_config.libraries,
 runtime_library_dirs=ext_config.runtime_library_dirs,
 extra_objects=ext_config.extra_objects,
 extra_compile_args=ext_config.extra_compile_args,
 extra_link_args=ext_config.extra_link_args,
 )
 ##print(ext.__dict__); sys.exit(1)
 return ext


###############################################################################

# prepare data_files

def get_data_files():
 # a list of tuples with (path to install to, a list of local files)
 data_files = []
 if sys.platform == "win32":
 datadir = os.path.join("doc", PACKAGE)
 else:
 datadir = os.path.join("share", "doc", PACKAGE)
 #
 files = ["AUTHORS", "ChangeLog", "COPYING-LGPL", "COPYING-MIT",
 "INSTALL.rst", "README.rst", "RELEASE-NOTES.rst"]
 if files:
 data_files.append((os.path.join(datadir), files))
 files = glob.glob(os.path.join("examples", "*.py"))
 if files:
 data_files.append((os.path.join(datadir, "examples"), files))
 files = glob.glob(os.path.join("examples", "quickstart", "*.py"))
 if files:
 data_files.append((os.path.join(datadir, "examples", "quickstart"), files))
 #
 assert data_files
 for install_dir, files in data_files:
 assert files
 for f in files:
 assert os.path.isfile(f), (f, install_dir)
 return data_files


###############################################################################

def check_manifest():
 import fnmatch
 
 f = open('MANIFEST.in')
 globs = []
 try:
 for line in f.readlines():
 stripped = line.strip()
 if stripped == '' or stripped.startswith('#'):
 continue
 assert stripped.startswith('include ')
 glob = stripped[8:]
 globs.append(glob)
 finally:
 f.close()
 
 paths = []
 start = os.path.abspath(os.path.dirname(__file__))
 for root, dirs, files in os.walk(start):
 if '.git' in dirs:
 dirs.remove('.git')
 for file in files:
 if file.endswith('.pyc'):
 continue
 rel = os.path.join(root, file)[len(start)+1:]
 paths.append(rel)
 
 for path in paths:
 included = False
 for glob in globs:
 if fnmatch.fnmatch(path, glob):
 included = True
 break
 if not included:
 print(path)

AUTHORS_PARAGRAPH = 3

def check_authors():
 f = open('AUTHORS')
 try:
 contents = f.read()
 finally:
 f.close()
 
 paras = contents.split("\n\n")
 authors_para = paras[AUTHORS_PARAGRAPH]
 authors = [author for author in authors_para.strip().split("\n")]
 
 log = subprocess.check_output(['git', 'log', '--format=%an (%ae)'])
 for author in log.strip().split("\n"):
 author = author.replace('@', ' at ').replace('(', '<').replace(')', '>')
 if author not in authors:
 authors.append(author)
 authors.sort()
 paras[AUTHORS_PARAGRAPH] = "\n".join(authors)
 f = open('AUTHORS', 'w')
 try:
 f.write("\n\n".join(paras))
 finally:
 f.close()


def convert_docstrings():
 docstrings = []
 for entry in sorted(os.listdir('doc/docstrings')):
 if not entry.endswith('.rst'):
 continue
 
 name = entry.replace('.rst', '')
 f = open('doc/docstrings/%s' % entry)
 try:
 text = f.read().strip()
 finally:
 f.close()
 docstrings.append((name, text))
 f = open('src/docstrings.c', 'w')
 try:
 f.write("/* Generated file - do not edit. */\n")
 # space to avoid having /* inside a C comment
 f.write("/* See doc/docstrings/ *.rst. */\n\n")
 f.write("#include \"pycurl.h\"\n\n")
 for name, text in docstrings:
 text = text.replace("\"", "\\\"").replace("\n", "\\n\\\n")
 f.write("PYCURL_INTERNAL const char %s_doc[] = \"%s\";\n\n" % (name, text))
 finally:
 f.close()
 f = open('src/docstrings.h', 'w')
 try:
 f.write("/* Generated file - do not edit. */\n")
 # space to avoid having /* inside a C comment
 f.write("/* See doc/docstrings/ *.rst. */\n\n")
 for name, text in docstrings:
 f.write("extern const char %s_doc[];\n" % name)
 finally:
 f.close()


def gen_docstrings_sources():
 sources = 'DOCSTRINGS_SOURCES ='
 for entry in sorted(os.listdir('doc/docstrings')):
 if entry.endswith('.rst'):
 sources += " \\\n\tdoc/docstrings/%s" % entry
 print(sources)

###############################################################################

setup_args = dict(
 name=PACKAGE,
 version=VERSION,
 description="PycURL -- cURL library module for Python",
 author="Kjetil Jacobsen, Markus F.X.J. Oberhumer, Oleg Pudeyev",
 author_email="kjetilja at gmail.com, markus at oberhumer.com, oleg at bsdpower.com",
 maintainer="Oleg Pudeyev",
 maintainer_email="oleg@bsdpower.com",
 url="http://pycurl.sourceforge.net/",
 download_url="http://pycurl.sourceforge.net/download/",
 license="LGPL/MIT",
 keywords=['curl', 'libcurl', 'urllib', 'wget', 'download', 'file transfer',
 'http', 'www'],
 classifiers=[
 'Development Status :: 5 - Production/Stable',
 'Environment :: Web Environment',
 'Intended Audience :: Developers',
 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)',
 'License :: OSI Approved :: MIT License',
 'Operating System :: Microsoft :: Windows',
 'Operating System :: POSIX',
 'Programming Language :: Python :: 2',
 'Programming Language :: Python :: 3',
 'Topic :: Internet :: File Transfer Protocol (FTP)',
 'Topic :: Internet :: WWW/HTTP',
 ],
 packages=[PY_PACKAGE],
 package_dir={ PY_PACKAGE: os.path.join('python', 'curl') },
 long_description="""
This module provides Python bindings for the cURL library.""",
)

if sys.platform == "win32":
 setup_args['cmdclass'] = {'bdist_msi': get_bdist_msi_version_hack()}

##print distutils.__version__
if LooseVersion(distutils.__version__) > LooseVersion("1.0.1"):
 setup_args["platforms"] = "All"
if LooseVersion(distutils.__version__) < LooseVersion("1.0.3"):
 setup_args["licence"] = setup_args["license"]

unix_help = '''\
PycURL Unix options:
 --curl-config=/path/to/curl-config use specified curl-config binary
 --openssl-dir=/path/to/openssl/dir path to OpenSSL headers and libraries
 --with-ssl libcurl is linked against OpenSSL
 --with-gnutls libcurl is linked against GnuTLS
 --with-nss libcurl is linked against NSS
'''

windows_help = '''\
PycURL Windows options:
 --curl-dir=/path/to/compiled/libcurl path to libcurl headers and libraries
 --use-libcurl-dll link against libcurl DLL, if not given
 link against libcurl statically
 --libcurl-lib-name=libcurl_imp.lib override libcurl import library name
'''

if __name__ == "__main__":
 if '--help' in sys.argv or '-h' in sys.argv:
 # unfortunately this help precedes distutils help
 if sys.platform == "win32":
 print(windows_help)
 else:
 print(unix_help)
 # invoke setup without configuring pycurl because
 # configuration might fail, and we want to display help anyway.
 # we need to remove our options because distutils complains about them
 strip_pycurl_options()
 setup(**setup_args)
 elif len(sys.argv) > 1 and sys.argv[1] == 'manifest':
 check_manifest()
 elif len(sys.argv) > 1 and sys.argv[1] == 'docstrings':
 convert_docstrings()
 elif len(sys.argv) > 1 and sys.argv[1] == 'authors':
 check_authors()
 elif len(sys.argv) > 1 and sys.argv[1] == 'docstrings-sources':
 gen_docstrings_sources()
 else:
 setup_args['data_files'] = get_data_files()
 if 'PYCURL_RELEASE' in os.environ and os.environ['PYCURL_RELEASE'].lower() in ['1', 'yes', 'true']:
 split_extension_source = False
 else:
 split_extension_source = True
 ext = get_extension(split_extension_source=split_extension_source)
 setup_args['ext_modules'] = [ext]
 
 for o in ext.extra_objects:
 assert os.path.isfile(o), o
 setup(**setup_args)