"""Conversion pipeline templates.

[0] Fix | Delete

[1] Fix | Delete

The problem:

[2] Fix | Delete

------------

[3] Fix | Delete

[4] Fix | Delete

Suppose you have some data that you want to convert to another format,

[5] Fix | Delete

such as from GIF image format to PPM image format. Maybe the

[6] Fix | Delete

conversion involves several steps (e.g. piping it through compress or

[7] Fix | Delete

uuencode). Some of the conversion steps may require that their input

[8] Fix | Delete

is a disk file, others may be able to read standard input; similar for

[9] Fix | Delete

their output. The input to the entire conversion may also be read

[10] Fix | Delete

from a disk file or from an open file, and similar for its output.

[11] Fix | Delete

[12] Fix | Delete

The module lets you construct a pipeline template by sticking one or

[13] Fix | Delete

more conversion steps together. It will take care of creating and

[14] Fix | Delete

removing temporary files if they are necessary to hold intermediate

[15] Fix | Delete

data. You can then use the template to do conversions from many

[16] Fix | Delete

different sources to many different destinations. The temporary

[17] Fix | Delete

file names used are different each time the template is used.

[18] Fix | Delete

[19] Fix | Delete

The templates are objects so you can create templates for many

[20] Fix | Delete

different conversion steps and store them in a dictionary, for

[21] Fix | Delete

instance.

[22] Fix | Delete

[23] Fix | Delete

[24] Fix | Delete

Directions:

[25] Fix | Delete

-----------

[26] Fix | Delete

[27] Fix | Delete

To create a template:

[28] Fix | Delete

t = Template()

[29] Fix | Delete

[30] Fix | Delete

To add a conversion step to a template:

[31] Fix | Delete

t.append(command, kind)

[32] Fix | Delete

where kind is a string of two characters: the first is '-' if the

[33] Fix | Delete

command reads its standard input or 'f' if it requires a file; the

[34] Fix | Delete

second likewise for the output. The command must be valid /bin/sh

[35] Fix | Delete

syntax. If input or output files are required, they are passed as

[36] Fix | Delete

$IN and $OUT; otherwise, it must be possible to use the command in

[37] Fix | Delete

a pipeline.

[38] Fix | Delete

[39] Fix | Delete

To add a conversion step at the beginning:

[40] Fix | Delete

t.prepend(command, kind)

[41] Fix | Delete

[42] Fix | Delete

To convert a file to another file using a template:

[43] Fix | Delete

sts = t.copy(infile, outfile)

[44] Fix | Delete

If infile or outfile are the empty string, standard input is read or

[45] Fix | Delete

standard output is written, respectively. The return value is the

[46] Fix | Delete

exit status of the conversion pipeline.

[47] Fix | Delete

[48] Fix | Delete

To open a file for reading or writing through a conversion pipeline:

[49] Fix | Delete

fp = t.open(file, mode)

[50] Fix | Delete

where mode is 'r' to read the file, or 'w' to write it -- just like

[51] Fix | Delete

for the built-in function open() or for os.popen().

[52] Fix | Delete

[53] Fix | Delete

To create a new template object initialized to a given one:

[54] Fix | Delete

t2 = t.clone()

[55] Fix | Delete

""" # '

[56] Fix | Delete

[57] Fix | Delete

[58] Fix | Delete

import re

[59] Fix | Delete

import os

[60] Fix | Delete

import tempfile

[61] Fix | Delete

import string

[62] Fix | Delete

[63] Fix | Delete

__all__ = ["Template"]

[64] Fix | Delete

[65] Fix | Delete

# Conversion step kinds

[66] Fix | Delete

[67] Fix | Delete

FILEIN_FILEOUT = 'ff' # Must read & write real files

[68] Fix | Delete

STDIN_FILEOUT = '-f' # Must write a real file

[69] Fix | Delete

FILEIN_STDOUT = 'f-' # Must read a real file

[70] Fix | Delete

STDIN_STDOUT = '--' # Normal pipeline element

[71] Fix | Delete

SOURCE = '.-' # Must be first, writes stdout

[72] Fix | Delete

SINK = '-.' # Must be last, reads stdin

[73] Fix | Delete

[74] Fix | Delete

stepkinds = [FILEIN_FILEOUT, STDIN_FILEOUT, FILEIN_STDOUT, STDIN_STDOUT, \

[75] Fix | Delete

SOURCE, SINK]

[76] Fix | Delete

[77] Fix | Delete

[78] Fix | Delete

class Template:

[79] Fix | Delete

"""Class representing a pipeline template."""

[80] Fix | Delete

[81] Fix | Delete

def __init__(self):

[82] Fix | Delete

"""Template() returns a fresh pipeline template."""

[83] Fix | Delete

self.debugging = 0

[84] Fix | Delete

self.reset()

[85] Fix | Delete

[86] Fix | Delete

def __repr__(self):

[87] Fix | Delete

"""t.__repr__() implements repr(t)."""

[88] Fix | Delete

return '<Template instance, steps=%r>' % (self.steps,)

[89] Fix | Delete

[90] Fix | Delete

def reset(self):

[91] Fix | Delete

"""t.reset() restores a pipeline template to its initial state."""

[92] Fix | Delete

self.steps = []

[93] Fix | Delete

[94] Fix | Delete

def clone(self):

[95] Fix | Delete

"""t.clone() returns a new pipeline template with identical

[96] Fix | Delete

initial state as the current one."""

[97] Fix | Delete

t = Template()

[98] Fix | Delete

t.steps = self.steps[:]

[99] Fix | Delete

t.debugging = self.debugging

[100] Fix | Delete

return t

[101] Fix | Delete

[102] Fix | Delete

def debug(self, flag):

[103] Fix | Delete

"""t.debug(flag) turns debugging on or off."""

[104] Fix | Delete

self.debugging = flag

[105] Fix | Delete

[106] Fix | Delete

def append(self, cmd, kind):

[107] Fix | Delete

"""t.append(cmd, kind) adds a new step at the end."""

[108] Fix | Delete

if type(cmd) is not type(''):

[109] Fix | Delete

raise TypeError, \

[110] Fix | Delete

'Template.append: cmd must be a string'

[111] Fix | Delete

if kind not in stepkinds:

[112] Fix | Delete

raise ValueError, \

[113] Fix | Delete

'Template.append: bad kind %r' % (kind,)

[114] Fix | Delete

if kind == SOURCE:

[115] Fix | Delete

raise ValueError, \

[116] Fix | Delete

'Template.append: SOURCE can only be prepended'

[117] Fix | Delete

if self.steps and self.steps[-1][1] == SINK:

[118] Fix | Delete

raise ValueError, \

[119] Fix | Delete

'Template.append: already ends with SINK'

[120] Fix | Delete

if kind[0] == 'f' and not re.search(r'\$IN\b', cmd):

[121] Fix | Delete

raise ValueError, \

[122] Fix | Delete

'Template.append: missing $IN in cmd'

[123] Fix | Delete

if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd):

[124] Fix | Delete

raise ValueError, \

[125] Fix | Delete

'Template.append: missing $OUT in cmd'

[126] Fix | Delete

self.steps.append((cmd, kind))

[127] Fix | Delete

[128] Fix | Delete

def prepend(self, cmd, kind):

[129] Fix | Delete

"""t.prepend(cmd, kind) adds a new step at the front."""

[130] Fix | Delete

if type(cmd) is not type(''):

[131] Fix | Delete

raise TypeError, \

[132] Fix | Delete

'Template.prepend: cmd must be a string'

[133] Fix | Delete

if kind not in stepkinds:

[134] Fix | Delete

raise ValueError, \

[135] Fix | Delete

'Template.prepend: bad kind %r' % (kind,)

[136] Fix | Delete

if kind == SINK:

[137] Fix | Delete

raise ValueError, \

[138] Fix | Delete

'Template.prepend: SINK can only be appended'

[139] Fix | Delete

if self.steps and self.steps[0][1] == SOURCE:

[140] Fix | Delete

raise ValueError, \

[141] Fix | Delete

'Template.prepend: already begins with SOURCE'

[142] Fix | Delete

if kind[0] == 'f' and not re.search(r'\$IN\b', cmd):

[143] Fix | Delete

raise ValueError, \

[144] Fix | Delete

'Template.prepend: missing $IN in cmd'

[145] Fix | Delete

if kind[1] == 'f' and not re.search(r'\$OUT\b', cmd):

[146] Fix | Delete

raise ValueError, \

[147] Fix | Delete

'Template.prepend: missing $OUT in cmd'

[148] Fix | Delete

self.steps.insert(0, (cmd, kind))

[149] Fix | Delete

[150] Fix | Delete

def open(self, file, rw):

[151] Fix | Delete

"""t.open(file, rw) returns a pipe or file object open for

[152] Fix | Delete

reading or writing; the file is the other end of the pipeline."""

[153] Fix | Delete

if rw == 'r':

[154] Fix | Delete

return self.open_r(file)

[155] Fix | Delete

if rw == 'w':

[156] Fix | Delete

return self.open_w(file)

[157] Fix | Delete

raise ValueError, \

[158] Fix | Delete

'Template.open: rw must be \'r\' or \'w\', not %r' % (rw,)

[159] Fix | Delete

[160] Fix | Delete

def open_r(self, file):

[161] Fix | Delete

"""t.open_r(file) and t.open_w(file) implement

[162] Fix | Delete

t.open(file, 'r') and t.open(file, 'w') respectively."""

[163] Fix | Delete

if not self.steps:

[164] Fix | Delete

return open(file, 'r')

[165] Fix | Delete

if self.steps[-1][1] == SINK:

[166] Fix | Delete

raise ValueError, \

[167] Fix | Delete

'Template.open_r: pipeline ends width SINK'

[168] Fix | Delete

cmd = self.makepipeline(file, '')

[169] Fix | Delete

return os.popen(cmd, 'r')

[170] Fix | Delete

[171] Fix | Delete

def open_w(self, file):

[172] Fix | Delete

if not self.steps:

[173] Fix | Delete

return open(file, 'w')

[174] Fix | Delete

if self.steps[0][1] == SOURCE:

[175] Fix | Delete

raise ValueError, \

[176] Fix | Delete

'Template.open_w: pipeline begins with SOURCE'

[177] Fix | Delete

cmd = self.makepipeline('', file)

[178] Fix | Delete

return os.popen(cmd, 'w')

[179] Fix | Delete

[180] Fix | Delete

def copy(self, infile, outfile):

[181] Fix | Delete

return os.system(self.makepipeline(infile, outfile))

[182] Fix | Delete

[183] Fix | Delete

def makepipeline(self, infile, outfile):

[184] Fix | Delete

cmd = makepipeline(infile, self.steps, outfile)

[185] Fix | Delete

if self.debugging:

[186] Fix | Delete

print cmd

[187] Fix | Delete

cmd = 'set -x; ' + cmd

[188] Fix | Delete

return cmd

[189] Fix | Delete

[190] Fix | Delete

[191] Fix | Delete

def makepipeline(infile, steps, outfile):

[192] Fix | Delete

# Build a list with for each command:

[193] Fix | Delete

# [input filename or '', command string, kind, output filename or '']

[194] Fix | Delete

[195] Fix | Delete

list = []

[196] Fix | Delete

for cmd, kind in steps:

[197] Fix | Delete

list.append(['', cmd, kind, ''])

[198] Fix | Delete

#

[199] Fix | Delete

# Make sure there is at least one step

[200] Fix | Delete

#

[201] Fix | Delete

if not list:

[202] Fix | Delete

list.append(['', 'cat', '--', ''])

[203] Fix | Delete

#

[204] Fix | Delete

# Take care of the input and output ends

[205] Fix | Delete

#

[206] Fix | Delete

[cmd, kind] = list[0][1:3]

[207] Fix | Delete

if kind[0] == 'f' and not infile:

[208] Fix | Delete

list.insert(0, ['', 'cat', '--', ''])

[209] Fix | Delete

list[0][0] = infile

[210] Fix | Delete

#

[211] Fix | Delete

[cmd, kind] = list[-1][1:3]

[212] Fix | Delete

if kind[1] == 'f' and not outfile:

[213] Fix | Delete

list.append(['', 'cat', '--', ''])

[214] Fix | Delete

list[-1][-1] = outfile

[215] Fix | Delete

#

[216] Fix | Delete

# Invent temporary files to connect stages that need files

[217] Fix | Delete

#

[218] Fix | Delete

garbage = []

[219] Fix | Delete

for i in range(1, len(list)):

[220] Fix | Delete

lkind = list[i-1][2]

[221] Fix | Delete

rkind = list[i][2]

[222] Fix | Delete

if lkind[1] == 'f' or rkind[0] == 'f':

[223] Fix | Delete

(fd, temp) = tempfile.mkstemp()

[224] Fix | Delete

os.close(fd)

[225] Fix | Delete

garbage.append(temp)

[226] Fix | Delete

list[i-1][-1] = list[i][0] = temp

[227] Fix | Delete

#

[228] Fix | Delete

for item in list:

[229] Fix | Delete

[inf, cmd, kind, outf] = item

[230] Fix | Delete

if kind[1] == 'f':

[231] Fix | Delete

cmd = 'OUT=' + quote(outf) + '; ' + cmd

[232] Fix | Delete

if kind[0] == 'f':

[233] Fix | Delete

cmd = 'IN=' + quote(inf) + '; ' + cmd

[234] Fix | Delete

if kind[0] == '-' and inf:

[235] Fix | Delete

cmd = cmd + ' <' + quote(inf)

[236] Fix | Delete

if kind[1] == '-' and outf:

[237] Fix | Delete

cmd = cmd + ' >' + quote(outf)

[238] Fix | Delete

item[1] = cmd

[239] Fix | Delete

#

[240] Fix | Delete

cmdlist = list[0][1]

[241] Fix | Delete

for item in list[1:]:

[242] Fix | Delete

[cmd, kind] = item[1:3]

[243] Fix | Delete

if item[0] == '':

[244] Fix | Delete

if 'f' in kind:

[245] Fix | Delete

cmd = '{ ' + cmd + '; }'

[246] Fix | Delete

cmdlist = cmdlist + ' |\n' + cmd

[247] Fix | Delete

else:

[248] Fix | Delete

cmdlist = cmdlist + '\n' + cmd

[249] Fix | Delete

#

[250] Fix | Delete

if garbage:

[251] Fix | Delete

rmcmd = 'rm -f'

[252] Fix | Delete

for file in garbage:

[253] Fix | Delete

rmcmd = rmcmd + ' ' + quote(file)

[254] Fix | Delete

trapcmd = 'trap ' + quote(rmcmd + '; exit') + ' 1 2 3 13 14 15'

[255] Fix | Delete

cmdlist = trapcmd + '\n' + cmdlist + '\n' + rmcmd

[256] Fix | Delete

#

[257] Fix | Delete

return cmdlist

[258] Fix | Delete

[259] Fix | Delete

[260] Fix | Delete

# Reliably quote a string as a single argument for /bin/sh

[261] Fix | Delete

[262] Fix | Delete

# Safe unquoted

[263] Fix | Delete

_safechars = frozenset(string.ascii_letters + string.digits + '@%_-+=:,./')

[264] Fix | Delete

[265] Fix | Delete

def quote(file):

[266] Fix | Delete

"""Return a shell-escaped version of the file string."""

[267] Fix | Delete

for c in file:

[268] Fix | Delete

if c not in _safechars:

[269] Fix | Delete

break

[270] Fix | Delete

else:

[271] Fix | Delete

if not file:

[272] Fix | Delete

return "''"

[273] Fix | Delete

return file

[274] Fix | Delete

# use single quotes, and put single quotes into double quotes

[275] Fix | Delete

# the string $'b is then quoted as '$'"'"'b'

[276] Fix | Delete

return "'" + file.replace("'", "'\"'\"'") + "'"

[277] Fix | Delete

[278] Fix | Delete