Viewing file:  _release.py (48.47 KB) -rw-r--r--
Select action/file-type:
 (+) |  (+) |  (+) | Code (+) | Session (+) |  (+) | SDB (+) |  (+) |  (+) |  (+) |  (+) |  (+) |
# -*- test-case-name: twisted.python.test.test_release -*-
# Copyright (c) 2007-2010 Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Twisted's automated release system.
This module is only for use within Twisted's release system. If you are anyone
else, do not use it. The interface and behaviour will change without notice.
Only Linux is supported by this code. It should not be used by any tools
which must run on multiple platforms (eg the setup.py script).
"""
import textwrap
from datetime import date
import re
import sys
import os
from tempfile import mkdtemp
import tarfile
from subprocess import PIPE, STDOUT, Popen
from twisted.python.versions import Version
from twisted.python.filepath import FilePath
from twisted.python.dist import twisted_subprojects
# This import is an example of why you shouldn't use this module unless you're
# radix
try:
from twisted.lore.scripts import lore
except ImportError:
pass
# The offset between a year and the corresponding major version number.
VERSION_OFFSET = 2000
# The list of subproject names to exclude from the main Twisted tarball and
# for which no individual project tarballs will be built.
PROJECT_BLACKLIST = ["vfs", "web2"]
def runCommand(args):
"""
Execute a vector of arguments.
@type args: C{list} of C{str}
@param args: A list of arguments, the first of which will be used as the
executable to run.
@rtype: C{str}
@return: All of the standard output.
@raise CommandFailed: when the program exited with a non-0 exit code.
"""
process = Popen(args, stdout=PIPE, stderr=STDOUT)
stdout = process.stdout.read()
exitCode = process.wait()
if exitCode < 0:
raise CommandFailed(None, -exitCode, stdout)
elif exitCode > 0:
raise CommandFailed(exitCode, None, stdout)
return stdout
class CommandFailed(Exception):
"""
Raised when a child process exits unsuccessfully.
@type exitStatus: C{int}
@ivar exitStatus: The exit status for the child process.
@type exitSignal: C{int}
@ivar exitSignal: The exit signal for the child process.
@type output: C{str}
@ivar output: The bytes read from stdout and stderr of the child process.
"""
def __init__(self, exitStatus, exitSignal, output):
Exception.__init__(self, exitStatus, exitSignal, output)
self.exitStatus = exitStatus
self.exitSignal = exitSignal
self.output = output
def _changeVersionInFile(old, new, filename):
"""
Replace the C{old} version number with the C{new} one in the given
C{filename}.
"""
replaceInFile(filename, {old.base(): new.base()})
def getNextVersion(version, now=None):
"""
Calculate the version number for a new release of Twisted based on
the previous version number.
@param version: The previous version number.
@param now: (optional) The current date.
"""
# XXX: This has no way of incrementing the patch number. Currently, we
# don't need it. See bug 2915. Jonathan Lange, 2007-11-20.
if now is None:
now = date.today()
major = now.year - VERSION_OFFSET
if major != version.major:
minor = 0
else:
minor = version.minor + 1
return Version(version.package, major, minor, 0)
def changeAllProjectVersions(root, versionTemplate, today=None):
"""
Change the version of all projects (including core and all subprojects).
If the current version of a project is pre-release, then also change the
versions in the current NEWS entries for that project.
@type root: L{FilePath}
@param root: The root of the Twisted source tree.
@type versionTemplate: L{Version}
@param versionTemplate: The version of all projects. The name will be
replaced for each respective project.
@type today: C{str}
@param today: A YYYY-MM-DD formatted string. If not provided, defaults to
the current day, according to the system clock.
"""
if not today:
today = date.today().strftime('%Y-%m-%d')
for project in findTwistedProjects(root):
if project.directory.basename() == "twisted":
packageName = "twisted"
else:
packageName = "twisted." + project.directory.basename()
oldVersion = project.getVersion()
newVersion = Version(packageName, versionTemplate.major,
versionTemplate.minor, versionTemplate.micro,
prerelease=versionTemplate.prerelease)
if oldVersion.prerelease:
builder = NewsBuilder()
builder._changeNewsVersion(
root.child("NEWS"), builder._getNewsName(project),
oldVersion, newVersion, today)
builder._changeNewsVersion(
project.directory.child("topfiles").child("NEWS"),
builder._getNewsName(project), oldVersion, newVersion,
today)
# The placement of the top-level README with respect to other files (eg
# _version.py) is sufficiently different from the others that we just
# have to handle it specially.
if packageName == "twisted":
_changeVersionInFile(
oldVersion, newVersion, root.child('README').path)
project.updateVersion(newVersion)
class Project(object):
"""
A representation of a project that has a version.
@ivar directory: A L{twisted.python.filepath.FilePath} pointing to the base
directory of a Twisted-style Python package. The package should contain
a C{_version.py} file and a C{topfiles} directory that contains a
C{README} file.
"""
def __init__(self, directory):
self.directory = directory
def __repr__(self):
return '%s(%r)' % (
self.__class__.__name__, self.directory)
def getVersion(self):
"""
@return: A L{Version} specifying the version number of the project
based on live python modules.
"""
namespace = {}
execfile(self.directory.child("_version.py").path, namespace)
return namespace["version"]
def updateVersion(self, version):
"""
Replace the existing version numbers in _version.py and README files
with the specified version.
"""
oldVersion = self.getVersion()
replaceProjectVersion(self.directory.child("_version.py").path,
version)
_changeVersionInFile(
oldVersion, version,
self.directory.child("topfiles").child("README").path)
def findTwistedProjects(baseDirectory):
"""
Find all Twisted-style projects beneath a base directory.
@param baseDirectory: A L{twisted.python.filepath.FilePath} to look inside.
@return: A list of L{Project}.
"""
projects = []
for filePath in baseDirectory.walk():
if filePath.basename() == 'topfiles':
projectDirectory = filePath.parent()
projects.append(Project(projectDirectory))
return projects
def updateTwistedVersionInformation(baseDirectory, now):
"""
Update the version information for Twisted and all subprojects to the
date-based version number.
@param baseDirectory: Where to look for Twisted. If None, the function
infers the information from C{twisted.__file__}.
@param now: The current date (as L{datetime.date}). If None, it defaults
to today.
"""
for project in findTwistedProjects(baseDirectory):
project.updateVersion(getNextVersion(project.getVersion(), now=now))
def generateVersionFileData(version):
"""
Generate the data to be placed into a _version.py file.
@param version: A version object.
"""
if version.prerelease is not None:
prerelease = ", prerelease=%r" % (version.prerelease,)
else:
prerelease = ""
data = '''\
# This is an auto-generated file. Do not edit it.
from twisted.python import versions
version = versions.Version(%r, %s, %s, %s%s)
''' % (version.package, version.major, version.minor, version.micro, prerelease)
return data
def replaceProjectVersion(filename, newversion):
"""
Write version specification code into the given filename, which
sets the version to the given version number.
@param filename: A filename which is most likely a "_version.py"
under some Twisted project.
@param newversion: A version object.
"""
# XXX - this should be moved to Project and renamed to writeVersionFile.
# jml, 2007-11-15.
f = open(filename, 'w')
f.write(generateVersionFileData(newversion))
f.close()
def replaceInFile(filename, oldToNew):
"""
I replace the text `oldstr' with `newstr' in `filename' using science.
"""
os.rename(filename, filename+'.bak')
f = open(filename+'.bak')
d = f.read()
f.close()
for k,v in oldToNew.items():
d = d.replace(k, v)
f = open(filename + '.new', 'w')
f.write(d)
f.close()
os.rename(filename+'.new', filename)
os.unlink(filename+'.bak')
class NoDocumentsFound(Exception):
"""
Raised when no input documents are found.
"""
class LoreBuilderMixin(object):
"""
Base class for builders which invoke lore.
"""
def lore(self, arguments):
"""
Run lore with the given arguments.
@param arguments: A C{list} of C{str} giving command line arguments to
lore which should be used.
"""
options = lore.Options()
options.parseOptions(["--null"] + arguments)
lore.runGivenOptions(options)
class DocBuilder(LoreBuilderMixin):
"""
Generate HTML documentation for projects.
"""
def build(self, version, resourceDir, docDir, template, apiBaseURL=None,
deleteInput=False):
"""
Build the documentation in C{docDir} with Lore.
Input files ending in .xhtml will be considered. Output will written as
.html files.
@param version: the version of the documentation to pass to lore.
@type version: C{str}
@param resourceDir: The directory which contains the toplevel index and
stylesheet file for this section of documentation.
@type resourceDir: L{twisted.python.filepath.FilePath}
@param docDir: The directory of the documentation.
@type docDir: L{twisted.python.filepath.FilePath}
@param template: The template used to generate the documentation.
@type template: L{twisted.python.filepath.FilePath}
@type apiBaseURL: C{str} or C{NoneType}
@param apiBaseURL: A format string which will be interpolated with the
fully-qualified Python name for each API link. For example, to
generate the Twisted 8.0.0 documentation, pass
C{"http://twistedmatrix.com/documents/8.0.0/api/%s.html"}.
@param deleteInput: If True, the input documents will be deleted after
their output is generated.
@type deleteInput: C{bool}
@raise NoDocumentsFound: When there are no .xhtml files in the given
C{docDir}.
"""
linkrel = self.getLinkrel(resourceDir, docDir)
inputFiles = docDir.globChildren("*.xhtml")
filenames = [x.path for x in inputFiles]
if not filenames:
raise NoDocumentsFound("No input documents found in %s" % (docDir,))
if apiBaseURL is not None:
arguments = ["--config", "baseurl=" + apiBaseURL]
else:
arguments = []
arguments.extend(["--config", "template=%s" % (template.path,),
"--config", "ext=.html",
"--config", "version=%s" % (version,),
"--linkrel", linkrel] + filenames)
self.lore(arguments)
if deleteInput:
for inputFile in inputFiles:
inputFile.remove()
def getLinkrel(self, resourceDir, docDir):
"""
Calculate a value appropriate for Lore's --linkrel option.
Lore's --linkrel option defines how to 'find' documents that are
linked to from TEMPLATE files (NOT document bodies). That is, it's a
prefix for links ('a' and 'link') in the template.
@param resourceDir: The directory which contains the toplevel index and
stylesheet file for this section of documentation.
@type resourceDir: L{twisted.python.filepath.FilePath}
@param docDir: The directory containing documents that must link to
C{resourceDir}.
@type docDir: L{twisted.python.filepath.FilePath}
"""
if resourceDir != docDir:
return '/'.join(filePathDelta(docDir, resourceDir)) + "/"
else:
return ""
class ManBuilder(LoreBuilderMixin):
"""
Generate man pages of the different existing scripts.
"""
def build(self, manDir):
"""
Generate Lore input files from the man pages in C{manDir}.
Input files ending in .1 will be considered. Output will written as
-man.xhtml files.
@param manDir: The directory of the man pages.
@type manDir: L{twisted.python.filepath.FilePath}
@raise NoDocumentsFound: When there are no .1 files in the given
C{manDir}.
"""
inputFiles = manDir.globChildren("*.1")
filenames = [x.path for x in inputFiles]
if not filenames:
raise NoDocumentsFound("No manual pages found in %s" % (manDir,))
arguments = ["--input", "man",
"--output", "lore",
"--config", "ext=-man.xhtml"] + filenames
self.lore(arguments)
class APIBuilder(object):
"""
Generate API documentation from source files using
U{pydoctor<http://codespeak.net/~mwh/pydoctor/>}. This requires
pydoctor to be installed and usable (which means you won't be able to
use it with Python 2.3).
"""
def build(self, projectName, projectURL, sourceURL, packagePath,
outputPath):
"""
Call pydoctor's entry point with options which will generate HTML
documentation for the specified package's API.
@type projectName: C{str}
@param projectName: The name of the package for which to generate
documentation.
@type projectURL: C{str}
@param projectURL: The location (probably an HTTP URL) of the project
on the web.
@type sourceURL: C{str}
@param sourceURL: The location (probably an HTTP URL) of the root of
the source browser for the project.
@type packagePath: L{FilePath}
@param packagePath: The path to the top-level of the package named by
C{projectName}.
@type outputPath: L{FilePath}
@param outputPath: An existing directory to which the generated API
documentation will be written.
"""
from pydoctor.driver import main
main(
["--project-name", projectName,
"--project-url", projectURL,
"--system-class", "pydoctor.twistedmodel.TwistedSystem",
"--project-base-dir", packagePath.parent().path,
"--html-viewsource-base", sourceURL,
"--add-package", packagePath.path,
"--html-output", outputPath.path,
"--html-write-function-pages", "--quiet", "--make-html"])
class BookBuilder(LoreBuilderMixin):
"""
Generate the LaTeX and PDF documentation.
The book is built by assembling a number of LaTeX documents. Only the
overall document which describes how to assemble the documents is stored
in LaTeX in the source. The rest of the documentation is generated from
Lore input files. These are primarily XHTML files (of the particular
Lore subset), but man pages are stored in GROFF format. BookBuilder
expects all of its input to be Lore XHTML format, so L{ManBuilder}
should be invoked first if the man pages are to be included in the
result (this is determined by the book LaTeX definition file).
Therefore, a sample usage of BookBuilder may look something like this::
man = ManBuilder()
man.build(FilePath("doc/core/man"))
book = BookBuilder()
book.build(
FilePath('doc/core/howto'),
[FilePath('doc/core/howto'), FilePath('doc/core/howto/tutorial'),
FilePath('doc/core/man'), FilePath('doc/core/specifications')],
FilePath('doc/core/howto/book.tex'), FilePath('/tmp/book.pdf'))
"""
def run(self, command):
"""
Execute a command in a child process and return the output.
@type command: C{str}
@param command: The shell command to run.
@raise CommandFailed: If the child process exits with an error.
"""
return runCommand(command)
def buildTeX(self, howtoDir):
"""
Build LaTeX files for lore input files in the given directory.
Input files ending in .xhtml will be considered. Output will written as
.tex files.
@type howtoDir: L{FilePath}
@param howtoDir: A directory containing lore input files.
@raise ValueError: If C{howtoDir} does not exist.
"""
if not howtoDir.exists():
raise ValueError("%r does not exist." % (howtoDir.path,))
self.lore(
["--output", "latex",
"--config", "section"] +
[child.path for child in howtoDir.globChildren("*.xhtml")])
def buildPDF(self, bookPath, inputDirectory, outputPath):
"""
Build a PDF from the given a LaTeX book document.
@type bookPath: L{FilePath}
@param bookPath: The location of a LaTeX document defining a book.
@type inputDirectory: L{FilePath}
@param inputDirectory: The directory which the inputs of the book are
relative to.
@type outputPath: L{FilePath}
@param outputPath: The location to which to write the resulting book.
"""
if not bookPath.basename().endswith(".tex"):
raise ValueError("Book filename must end with .tex")
workPath = FilePath(mkdtemp())
try:
startDir = os.getcwd()
try:
os.chdir(inputDirectory.path)
texToDVI = [
"latex", "-interaction=nonstopmode",
"-output-directory=" + workPath.path,
bookPath.path]
# What I tell you three times is true!
# The first two invocations of latex on the book file allows it
# correctly create page numbers for in-text references. Why this is
# the case, I could not tell you. -exarkun
for i in range(3):
self.run(texToDVI)
bookBaseWithoutExtension = bookPath.basename()[:-4]
dviPath = workPath.child(bookBaseWithoutExtension + ".dvi")
psPath = workPath.child(bookBaseWithoutExtension + ".ps")
pdfPath = workPath.child(bookBaseWithoutExtension + ".pdf")
self.run([
"dvips", "-o", psPath.path, "-t", "letter", "-Ppdf",
dviPath.path])
self.run(["ps2pdf13", psPath.path, pdfPath.path])
pdfPath.moveTo(outputPath)
workPath.remove()
finally:
os.chdir(startDir)
except:
workPath.moveTo(bookPath.parent().child(workPath.basename()))
raise
def build(self, baseDirectory, inputDirectories, bookPath, outputPath):
"""
Build a PDF book from the given TeX book definition and directories
containing lore inputs.
@type baseDirectory: L{FilePath}
@param baseDirectory: The directory which the inputs of the book are
relative to.
@type inputDirectories: C{list} of L{FilePath}
@param inputDirectories: The paths which contain lore inputs to be
converted to LaTeX.
@type bookPath: L{FilePath}
@param bookPath: The location of a LaTeX document defining a book.
@type outputPath: L{FilePath}
@param outputPath: The location to which to write the resulting book.
"""
for inputDir in inputDirectories:
self.buildTeX(inputDir)
self.buildPDF(bookPath, baseDirectory, outputPath)
for inputDirectory in inputDirectories:
for child in inputDirectory.children():
if child.splitext()[1] == ".tex" and child != bookPath:
child.remove()
class NewsBuilder(object):
"""
Generate the new section of a NEWS file.
The C{_FEATURE}, C{_BUGFIX}, C{_DOC}, C{_REMOVAL}, and C{_MISC}
attributes of this class are symbolic names for the news entry types
which are supported. Conveniently, they each also take on the value of
the file name extension which indicates a news entry of that type.
@cvar blacklist: A C{list} of C{str} of projects for which we should not
generate news at all. Same as C{PROJECT_BLACKLIST}.
@cvar _headings: A C{dict} mapping one of the news entry types to the
heading to write out for that type of news entry.
@cvar _NO_CHANGES: A C{str} giving the text which appears when there are
no significant changes in a release.
@cvar _TICKET_HINT: A C{str} giving the text which appears at the top of
each news file and which should be kept at the top, not shifted down
with all the other content. Put another way, this is the text after
which the new news text is inserted.
"""
blacklist = PROJECT_BLACKLIST
_FEATURE = ".feature"
_BUGFIX = ".bugfix"
_DOC = ".doc"
_REMOVAL = ".removal"
_MISC = ".misc"
_headings = {
_FEATURE: "Features",
_BUGFIX: "Bugfixes",
_DOC: "Improved Documentation",
_REMOVAL: "Deprecations and Removals",
_MISC: "Other",
}
_NO_CHANGES = "No significant changes have been made for this release.\n"
_TICKET_HINT = (
'Ticket numbers in this file can be looked up by visiting\n'
'http://twistedmatrix.com/trac/ticket/<number>\n'
'\n')
def _today(self):
"""
Return today's date as a string in YYYY-MM-DD format.
"""
return date.today().strftime('%Y-%m-%d')
def _findChanges(self, path, ticketType):
"""
Load all the feature ticket summaries.
@param path: A L{FilePath} the direct children of which to search
for news entries.
@param ticketType: The type of news entries to search for. One of
L{NewsBuilder._FEATURE}, L{NewsBuilder._BUGFIX},
L{NewsBuilder._REMOVAL}, or L{NewsBuilder._MISC}.
@return: A C{list} of two-tuples. The first element is the ticket
number as an C{int}. The second element of each tuple is the
description of the feature.
"""
results = []
for child in path.children():
base, ext = os.path.splitext(child.basename())
if ext == ticketType:
results.append((
int(base),
' '.join(child.getContent().splitlines())))
results.sort()
return results
def _formatHeader(self, header):
"""
Format a header for a NEWS file.
A header is a title with '=' signs underlining it.
@param header: The header string to format.
@type header: C{str}
@return: A C{str} containing C{header}.
"""
return header + '\n' + '=' * len(header) + '\n\n'
def _writeHeader(self, fileObj, header):
"""
Write a version header to the given file.
@param fileObj: A file-like object to which to write the header.
@param header: The header to write to the file.
@type header: C{str}
"""
fileObj.write(self._formatHeader(header))
def _writeSection(self, fileObj, header, tickets):
"""
Write out one section (features, bug fixes, etc) to the given file.
@param fileObj: A file-like object to which to write the news section.
@param header: The header for the section to write.
@type header: C{str}
@param tickets: A C{list} of ticket information of the sort returned
by L{NewsBuilder._findChanges}.
"""
if not tickets:
return
reverse = {}
for (ticket, description) in tickets:
reverse.setdefault(description, []).append(ticket)
for description in reverse:
reverse[description].sort()
reverse = reverse.items()
reverse.sort(key=lambda (descr, tickets): tickets[0])
fileObj.write(header + '\n' + '-' * len(header) + '\n')
for (description, relatedTickets) in reverse:
ticketList = ', '.join([
'#' + str(ticket) for ticket in relatedTickets])
entry = ' - %s (%s)' % (description, ticketList)
entry = textwrap.fill(entry, subsequent_indent=' ')
fileObj.write(entry + '\n')
fileObj.write('\n')
def _writeMisc(self, fileObj, header, tickets):
"""
Write out a miscellaneous-changes section to the given file.
@param fileObj: A file-like object to which to write the news section.
@param header: The header for the section to write.
@type header: C{str}
@param tickets: A C{list} of ticket information of the sort returned
by L{NewsBuilder._findChanges}.
"""
if not tickets:
return
fileObj.write(header + '\n' + '-' * len(header) + '\n')
formattedTickets = []
for (ticket, ignored) in tickets:
formattedTickets.append('#' + str(ticket))
entry = ' - ' + ', '.join(formattedTickets)
entry = textwrap.fill(entry, subsequent_indent=' ')
fileObj.write(entry + '\n\n')
def build(self, path, output, header):
"""
Load all of the change information from the given directory and write
it out to the given output file.
@param path: A directory (probably a I{topfiles} directory) containing
change information in the form of <ticket>.<change type> files.
@type path: L{FilePath}
@param output: The NEWS file to which the results will be prepended.
@type output: L{FilePath}
@param header: The top-level header to use when writing the news.
@type header: L{str}
"""
changes = []
for part in (self._FEATURE, self._BUGFIX, self._DOC, self._REMOVAL):
tickets = self._findChanges(path, part)
if tickets:
changes.append((part, tickets))
misc = self._findChanges(path, self._MISC)
oldNews = output.getContent()
newNews = output.sibling('NEWS.new').open('w')
if oldNews.startswith(self._TICKET_HINT):
newNews.write(self._TICKET_HINT)
oldNews = oldNews[len(self._TICKET_HINT):]
self._writeHeader(newNews, header)
if changes:
for (part, tickets) in changes:
self._writeSection(newNews, self._headings.get(part), tickets)
else:
newNews.write(self._NO_CHANGES)
newNews.write('\n')
self._writeMisc(newNews, self._headings.get(self._MISC), misc)
newNews.write('\n')
newNews.write(oldNews)
newNews.close()
output.sibling('NEWS.new').moveTo(output)
def _getNewsName(self, project):
"""
Return the name of C{project} that should appear in NEWS.
@param project: A L{Project}
@return: The name of C{project}.
"""
name = project.directory.basename().title()
if name == 'Twisted':
name = 'Core'
return name
def _iterProjects(self, baseDirectory):
"""
Iterate through the Twisted projects in C{baseDirectory}, yielding
everything we need to know to build news for them.
Yields C{topfiles}, C{news}, C{name}, C{version} for each sub-project
in reverse-alphabetical order. C{topfile} is the L{FilePath} for the
topfiles directory, C{news} is the L{FilePath} for the NEWS file,
C{name} is the nice name of the project (as should appear in the NEWS
file), C{version} is the current version string for that project.
@param baseDirectory: A L{FilePath} representing the root directory
beneath which to find Twisted projects for which to generate
news (see L{findTwistedProjects}).
@type baseDirectory: L{FilePath}
"""
# Get all the subprojects to generate news for
projects = findTwistedProjects(baseDirectory)
# And order them alphabetically for ease of reading
projects.sort(key=lambda proj: proj.directory.path)
# And generate them backwards since we write news by prepending to
# files.
projects.reverse()
for aggregateNews in [False, True]:
for project in projects:
if project.directory.basename() in self.blacklist:
continue
topfiles = project.directory.child("topfiles")
if aggregateNews:
news = baseDirectory.child("NEWS")
else:
news = topfiles.child("NEWS")
name = self._getNewsName(project)
version = project.getVersion()
yield topfiles, news, name, version
def buildAll(self, baseDirectory):
"""
Find all of the Twisted subprojects beneath C{baseDirectory} and update
their news files from the ticket change description files in their
I{topfiles} directories and update the news file in C{baseDirectory}
with all of the news.
Projects that are listed in L{NewsBuilder.blacklist} will be skipped.
@param baseDirectory: A L{FilePath} representing the root directory
beneath which to find Twisted projects for which to generate
news (see L{findTwistedProjects}).
"""
today = self._today()
for topfiles, news, name, version in self._iterProjects(baseDirectory):
self.build(
topfiles, news,
"Twisted %s %s (%s)" % (name, version.base(), today))
def _changeNewsVersion(self, news, name, oldVersion, newVersion, today):
"""
Change all references to the current version number in a NEWS file to
refer to C{newVersion} instead.
@param news: The NEWS file to change.
@type news: L{FilePath}
@param name: The name of the project to change.
@type name: C{str}
@param oldVersion: The old version of the project.
@type oldVersion: L{Version}
@param newVersion: The new version of the project.
@type newVersion: L{Version}
@param today: A YYYY-MM-DD string representing today's date.
@type today: C{str}
"""
newHeader = self._formatHeader(
"Twisted %s %s (%s)" % (name, newVersion.base(), today))
expectedHeaderRegex = re.compile(
r"Twisted %s %s \(\d{4}-\d\d-\d\d\)\n=+\n\n" % (
re.escape(name), re.escape(oldVersion.base())))
oldNews = news.getContent()
match = expectedHeaderRegex.search(oldNews)
if match:
oldHeader = match.group()
replaceInFile(news.path, {oldHeader: newHeader})
def main(self, args):
"""
Build all news files.
@param args: The command line arguments to process. This must contain
one string, the path to the base of the Twisted checkout for which
to build the news.
@type args: C{list} of C{str}
"""
if len(args) != 1:
sys.exit("Must specify one argument: the path to the Twisted checkout")
self.buildAll(FilePath(args[0]))
def filePathDelta(origin, destination):
"""
Return a list of strings that represent C{destination} as a path relative
to C{origin}.
It is assumed that both paths represent directories, not files. That is to
say, the delta of L{twisted.python.filepath.FilePath} /foo/bar to
L{twisted.python.filepath.FilePath} /foo/baz will be C{../baz},
not C{baz}.
@type origin: L{twisted.python.filepath.FilePath}
@param origin: The origin of the relative path.
@type destination: L{twisted.python.filepath.FilePath}
@param destination: The destination of the relative path.
"""
commonItems = 0
path1 = origin.path.split(os.sep)
path2 = destination.path.split(os.sep)
for elem1, elem2 in zip(path1, path2):
if elem1 == elem2:
commonItems += 1
else:
break
path = [".."] * (len(path1) - commonItems)
return path + path2[commonItems:]
class DistributionBuilder(object):
"""
A builder of Twisted distributions.
This knows how to build tarballs for Twisted and all of its subprojects.
@type blacklist: C{list} of C{str}
@cvar blacklist: The list of subproject names to exclude from the main
Twisted tarball and for which no individual project tarballs will be
built. The same list as C{PROJECT_BLACKLIST}.
"""
from twisted.python.dist import twisted_subprojects as subprojects
blacklist = PROJECT_BLACKLIST
def __init__(self, rootDirectory, outputDirectory, apiBaseURL=None):
"""
Create a distribution builder.
@param rootDirectory: root of a Twisted export which will populate
subsequent tarballs.
@type rootDirectory: L{FilePath}.
@param outputDirectory: The directory in which to create the tarballs.
@type outputDirectory: L{FilePath}
@type apiBaseURL: C{str} or C{NoneType}
@param apiBaseURL: A format string which will be interpolated with the
fully-qualified Python name for each API link. For example, to
generate the Twisted 8.0.0 documentation, pass
C{"http://twistedmatrix.com/documents/8.0.0/api/%s.html"}.
"""
self.rootDirectory = rootDirectory
self.outputDirectory = outputDirectory
self.apiBaseURL = apiBaseURL
self.manBuilder = ManBuilder()
self.docBuilder = DocBuilder()
def _buildDocInDir(self, path, version, howtoPath):
"""
Generate documentation in the given path, building man pages first if
necessary and swallowing errors (so that directories without lore
documentation in them are ignored).
@param path: The path containing documentation to build.
@type path: L{FilePath}
@param version: The version of the project to include in all generated
pages.
@type version: C{str}
@param howtoPath: The "resource path" as L{DocBuilder} describes it.
@type howtoPath: L{FilePath}
"""
templatePath = self.rootDirectory.child("doc").child("core"
).child("howto").child("template.tpl")
if path.basename() == "man":
self.manBuilder.build(path)
if path.isdir():
try:
self.docBuilder.build(version, howtoPath, path,
templatePath, self.apiBaseURL, True)
except NoDocumentsFound:
pass
def buildTwisted(self, version):
"""
Build the main Twisted distribution in C{Twisted-<version>.tar.bz2}.
Projects listed in in L{blacklist} will not have their plugins, code,
documentation, or bin directories included.
bin/admin is also excluded.
@type version: C{str}
@param version: The version of Twisted to build.
@return: The tarball file.
@rtype: L{FilePath}.
"""
releaseName = "Twisted-%s" % (version,)
buildPath = lambda *args: '/'.join((releaseName,) + args)
outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
tarball = tarfile.TarFile.open(outputFile.path, 'w:bz2')
docPath = self.rootDirectory.child("doc")
# Generate docs!
if docPath.isdir():
for subProjectDir in docPath.children():
if (subProjectDir.isdir()
and subProjectDir.basename() not in self.blacklist):
for child in subProjectDir.walk():
self._buildDocInDir(child, version,
subProjectDir.child("howto"))
# Now, this part is nasty. We need to exclude blacklisted subprojects
# from the main Twisted distribution. This means we need to exclude
# their bin directories, their documentation directories, their
# plugins, and their python packages. Given that there's no "add all
# but exclude these particular paths" functionality in tarfile, we have
# to walk through all these directories and add things that *aren't*
# part of the blacklisted projects.
for binthing in self.rootDirectory.child("bin").children():
# bin/admin should also not be included.
if binthing.basename() not in self.blacklist + ["admin"]:
tarball.add(binthing.path,
buildPath("bin", binthing.basename()))
bad_plugins = ["twisted_%s.py" % (blacklisted,)
for blacklisted in self.blacklist]
for submodule in self.rootDirectory.child("twisted").children():
if submodule.basename() == "plugins":
for plugin in submodule.children():
if plugin.basename() not in bad_plugins:
tarball.add(plugin.path, buildPath("twisted", "plugins",
plugin.basename()))
elif submodule.basename() not in self.blacklist:
tarball.add(submodule.path, buildPath("twisted",
submodule.basename()))
for docDir in self.rootDirectory.child("doc").children():
if docDir.basename() not in self.blacklist:
tarball.add(docDir.path, buildPath("doc", docDir.basename()))
for toplevel in self.rootDirectory.children():
if not toplevel.isdir():
tarball.add(toplevel.path, buildPath(toplevel.basename()))
tarball.close()
return outputFile
def buildCore(self, version):
"""
Build a core distribution in C{TwistedCore-<version>.tar.bz2}.
This is very similar to L{buildSubProject}, but core tarballs and the
input are laid out slightly differently.
- scripts are in the top level of the C{bin} directory.
- code is included directly from the C{twisted} directory, excluding
subprojects.
- all plugins except the subproject plugins are included.
@type version: C{str}
@param version: The version of Twisted to build.
@return: The tarball file.
@rtype: L{FilePath}.
"""
releaseName = "TwistedCore-%s" % (version,)
outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
buildPath = lambda *args: '/'.join((releaseName,) + args)
tarball = self._createBasicSubprojectTarball(
"core", version, outputFile)
# Include the bin directory for the subproject.
for path in self.rootDirectory.child("bin").children():
if not path.isdir():
tarball.add(path.path, buildPath("bin", path.basename()))
# Include all files within twisted/ that aren't part of a subproject.
for path in self.rootDirectory.child("twisted").children():
if path.basename() == "plugins":
for plugin in path.children():
for subproject in self.subprojects:
if plugin.basename() == "twisted_%s.py" % (subproject,):
break
else:
tarball.add(plugin.path,
buildPath("twisted", "plugins",
plugin.basename()))
elif not path.basename() in self.subprojects + ["topfiles"]:
tarball.add(path.path, buildPath("twisted", path.basename()))
tarball.add(self.rootDirectory.child("twisted").child("topfiles").path,
releaseName)
tarball.close()
return outputFile
def buildSubProject(self, projectName, version):
"""
Build a subproject distribution in
C{Twisted<Projectname>-<version>.tar.bz2}.
@type projectName: C{str}
@param projectName: The lowercase name of the subproject to build.
@type version: C{str}
@param version: The version of Twisted to build.
@return: The tarball file.
@rtype: L{FilePath}.
"""
releaseName = "Twisted%s-%s" % (projectName.capitalize(), version)
outputFile = self.outputDirectory.child(releaseName + ".tar.bz2")
buildPath = lambda *args: '/'.join((releaseName,) + args)
subProjectDir = self.rootDirectory.child("twisted").child(projectName)
tarball = self._createBasicSubprojectTarball(projectName, version,
outputFile)
tarball.add(subProjectDir.child("topfiles").path, releaseName)
# Include all files in the subproject package except for topfiles.
for child in subProjectDir.children():
name = child.basename()
if name != "topfiles":
tarball.add(
child.path,
buildPath("twisted", projectName, name))
pluginsDir = self.rootDirectory.child("twisted").child("plugins")
# Include the plugin for the subproject.
pluginFileName = "twisted_%s.py" % (projectName,)
pluginFile = pluginsDir.child(pluginFileName)
if pluginFile.exists():
tarball.add(pluginFile.path,
buildPath("twisted", "plugins", pluginFileName))
# Include the bin directory for the subproject.
binPath = self.rootDirectory.child("bin").child(projectName)
if binPath.isdir():
tarball.add(binPath.path, buildPath("bin"))
tarball.close()
return outputFile
def _createBasicSubprojectTarball(self, projectName, version, outputFile):
"""
Helper method to create and fill a tarball with things common between
subprojects and core.
@param projectName: The subproject's name.
@type projectName: C{str}
@param version: The version of the release.
@type version: C{str}
@param outputFile: The location of the tar file to create.
@type outputFile: L{FilePath}
"""
releaseName = "Twisted%s-%s" % (projectName.capitalize(), version)
buildPath = lambda *args: '/'.join((releaseName,) + args)
tarball = tarfile.TarFile.open(outputFile.path, 'w:bz2')
tarball.add(self.rootDirectory.child("LICENSE").path,
buildPath("LICENSE"))
docPath = self.rootDirectory.child("doc").child(projectName)
if docPath.isdir():
for child in docPath.walk():
self._buildDocInDir(child, version, docPath.child("howto"))
tarball.add(docPath.path, buildPath("doc"))
return tarball
class UncleanWorkingDirectory(Exception):
"""
Raised when the working directory of an SVN checkout is unclean.
"""
class NotWorkingDirectory(Exception):
"""
Raised when a directory does not appear to be an SVN working directory.
"""
def buildAllTarballs(checkout, destination):
"""
Build complete tarballs (including documentation) for Twisted and all
subprojects.
This should be called after the version numbers have been updated and
NEWS files created.
@type checkout: L{FilePath}
@param checkout: The SVN working copy from which a pristine source tree
will be exported.
@type destination: L{FilePath}
@param destination: The directory in which tarballs will be placed.
@raise UncleanWorkingDirectory: if there are modifications to the
working directory of C{checkout}.
@raise NotWorkingDirectory: if the checkout path is not an SVN checkout.
"""
if not checkout.child(".svn").exists():
raise NotWorkingDirectory(
"%s does not appear to be an SVN working directory."
% (checkout.path,))
if runCommand(["svn", "st", checkout.path]).strip():
raise UncleanWorkingDirectory(
"There are local modifications to the SVN checkout in %s."
% (checkout.path,))
workPath = FilePath(mkdtemp())
export = workPath.child("export")
runCommand(["svn", "export", checkout.path, export.path])
twistedPath = export.child("twisted")
version = Project(twistedPath).getVersion()
versionString = version.base()
apiBaseURL = "http://twistedmatrix.com/documents/%s/api/%%s.html" % (
versionString)
if not destination.exists():
destination.createDirectory()
db = DistributionBuilder(export, destination, apiBaseURL=apiBaseURL)
db.buildCore(versionString)
for subproject in twisted_subprojects:
if (subproject not in db.blacklist
and twistedPath.child(subproject).exists()):
db.buildSubProject(subproject, versionString)
db.buildTwisted(versionString)
workPath.remove()
class ChangeVersionsScript(object):
"""
A thing for changing version numbers. See L{main}.
"""
changeAllProjectVersions = staticmethod(changeAllProjectVersions)
def main(self, args):
"""
Given a list of command-line arguments, change all the Twisted versions
in the current directory.
@type args: list of str
@param args: List of command line arguments. This should only
contain the version number.
"""
version_format = (
"Version should be in a form kind of like '1.2.3[pre4]'")
if len(args) != 1:
sys.exit("Must specify exactly one argument to change-versions")
version = args[0]
try:
major, minor, micro_and_pre = version.split(".")
except ValueError:
raise SystemExit(version_format)
if "pre" in micro_and_pre:
micro, pre = micro_and_pre.split("pre")
else:
micro = micro_and_pre
pre = None
try:
major = int(major)
minor = int(minor)
micro = int(micro)
if pre is not None:
pre = int(pre)
except ValueError:
raise SystemExit(version_format)
version_template = Version("Whatever",
major, minor, micro, prerelease=pre)
self.changeAllProjectVersions(FilePath("."), version_template)
class BuildTarballsScript(object):
"""
A thing for building release tarballs. See L{main}.
"""
buildAllTarballs = staticmethod(buildAllTarballs)
def main(self, args):
"""
Build all release tarballs.
@type args: list of str
@param args: The command line arguments to process. This must contain
two strings: the checkout directory and the destination directory.
"""
if len(args) != 2:
sys.exit("Must specify two arguments: "
"Twisted checkout and destination path")
self.buildAllTarballs(FilePath(args[0]), FilePath(args[1]))
class BuildAPIDocsScript(object):
"""
A thing for building API documentation. See L{main}.
"""
def buildAPIDocs(self, projectRoot, output):
"""
Build the API documentation of Twisted, with our project policy.
@param projectRoot: A L{FilePath} representing the root of the Twisted
checkout.
@param output: A L{FilePath} pointing to the desired output directory.
"""
version = Project(projectRoot.child("twisted")).getVersion()
versionString = version.base()
sourceURL = ("http://twistedmatrix.com/trac/browser/tags/releases/"
"twisted-%s" % (versionString,))
apiBuilder = APIBuilder()
apiBuilder.build(
"Twisted",
"http://twistedmatrix.com/",
sourceURL,
projectRoot.child("twisted"),
output)
def main(self, args):
"""
Build API documentation.
@type args: list of str
@param args: The command line arguments to process. This must contain
two strings: the path to the root of the Twisted checkout, and a
path to an output directory.
"""
if len(args) != 2:
sys.exit("Must specify two arguments: "
"Twisted checkout and destination path")
self.buildAPIDocs(FilePath(args[0]), FilePath(args[1]))
|