1
2
3
4 """
5 General purpose utilities functions for fabio
6 """
7 from __future__ import with_statement
8 import re, os, logging, threading, sys
9 import StringIO as stringIO
10 logger = logging.getLogger("fabioutils")
11 from compression import bz2, gzip
12 import traceback
13
14
15
16 FILETYPES = {
17
18
19 'edf' : ['edf'],
20 'cor' : ['edf'],
21 'pnm' : ['pnm'],
22 'pgm' : ['pnm'],
23 'pbm' : ['pnm'],
24 'tif' : ['tif'],
25 'tiff' : ['tif'],
26 'img' : ['adsc', 'OXD', 'HiPiC'],
27 'mccd' : ['marccd'],
28 'mar2300': ['mar345'],
29 'sfrm' : ['bruker100'],
30 'msk' : ['fit2dmask'],
31 'spr' : ['fit2dspreadsheet'],
32 'dm3' : ['dm3'],
33 'kcd' : ['kcd'],
34 'cbf' : ['cbf'],
35 'xml' : ["xsd"],
36 'xsd' : ["xsd"],
37 }
38
39
40 for key in FILETYPES.keys():
41 FILETYPES[key + ".bz2"] = FILETYPES[key]
42 FILETYPES[key + ".gz"] = FILETYPES[key]
43
44
45
46
47 COMPRESSORS = {}
48
49
50 dictAscii = {None:[chr(i) for i in range(32, 127)],
51 }
52
53 try:
54 lines = os.popen("gzip -h 2>&1").read()
55
56 if "sage" in lines:
57 COMPRESSORS['.gz'] = 'gzip -dc '
58 else:
59 COMPRESSORS['.gz'] = None
60 except Exception:
61 COMPRESSORS['.gz'] = None
62
63 try:
64 lines = os.popen("bzip2 -h 2>&1").read()
65
66 if "sage" in lines:
67 COMPRESSORS['.bz2'] = 'bzip2 -dc '
68 else:
69 COMPRESSORS['.bz2'] = None
70 except Exception:
71 COMPRESSORS['.bz2'] = None
74 """
75 used to deprecate a function/method: prints a lot of warning messages to enforce the modifaction of the code
76 """
77 def wrapper(*arg, **kw):
78 """
79 decorator that deprecates the use of a function
80 """
81 logger.warning("%s is Deprecated !!! %s" % (func.func_name, os.linesep.join([""] + traceback.format_stack()[:-1])))
82 return func(*arg, **kw)
83 return wrapper
84
87 """
88 # try to figure out a file number
89 # guess it starts at the back
90 """
91 stem , num, post_num = numstem(name)
92 try:
93 return int(num)
94 except ValueError:
95 return None
96
98 """
99 The 'meaning' of a filename ...
100 """
101 - def __init__(self, stem=None,
102 num=None,
103 directory=None,
104 format=None,
105 extension=None,
106 postnum=None,
107 digits=4,
108 filename = None):
109 """
110 This class can either be instanciated by a set of parameters like directory, prefix, num, extension, ...
111
112 @param stem: the stem is a kind of prefix (str)
113 @param num: image number in the serie (int)
114 @param directory: name of the directory (str)
115 @param format: ??
116 @param extension:
117 @param postnum:
118 @param digits: Number of digits used to print num
119
120 Alternative constructor:
121
122 @param filename: fullpath of an image file to be deconstructed into directory, prefix, num, extension, ...
123
124 """
125
126
127 self.stem = stem
128 self.num = num
129 self.format = format
130 self.extension = extension
131 self.digits = digits
132 self.postnum = postnum
133 self.directory = directory
134 self.compressed = None
135 if filename is not None:
136 self.deconstruct_filename(filename)
137
138
140 """ Return a string representation """
141 fmt = "stem %s, num %s format %s extension %s " + \
142 "postnum = %s digits %s dir %s"
143 return fmt % tuple([str(x) for x in [
144 self.stem ,
145 self.num ,
146 self.format ,
147 self.extension ,
148 self.postnum ,
149 self.digits ,
150 self.directory ] ])
151 __repr__ = str
152
154 """
155 convert yourself to a string
156 """
157 name = self.stem
158 if self.digits is not None and self.num is not None:
159 fmt = "%0" + str(self.digits) + "d"
160 name += fmt % self.num
161 if self.postnum is not None:
162 name += self.postnum
163 if self.extension is not None:
164 name += self.extension
165 if self.directory is not None:
166 name = os.path.join(self.directory, name)
167 return name
168
169
171 """
172 Break up a filename to get image type and number
173 """
174 direc, name = os.path.split(filename)
175 direc = direc or None
176 parts = name.split(".")
177 compressed = False
178 stem = parts[0]
179 extn = ""
180 postnum = ""
181 ndigit = 4
182 num = None
183 typ = None
184 if parts[-1] in ["gz", "bz2"]:
185 extn = "." + parts[-1]
186 parts = parts[:-1]
187 compressed = True
188 if parts[-1] in FILETYPES.keys():
189 typ = FILETYPES[parts[-1]]
190 extn = "." + parts[-1] + extn
191 try:
192 stem, numstring, postnum = numstem(".".join(parts[:-1]))
193 num = int(numstring)
194 ndigit = len(numstring)
195 except Exception, err:
196
197 logger.debug("l176: %s" % err)
198 num = None
199 stem = "".join(parts[:-1])
200 else:
201
202 if len(parts) == 1:
203
204 parts2 = parts[0].split("_")
205 if parts2[-1].isdigit():
206 num = int(parts2[-1])
207 ndigit = len(parts2[-1])
208 typ = ['GE']
209 stem = "_".join(parts2[:-1]) + "_"
210 else:
211 try:
212 num = int(parts[-1])
213 ndigit = len(parts[-1])
214 typ = ['bruker']
215 stem = ".".join(parts[:-1]) + "."
216 except Exception, err:
217 logger.debug("l196: %s" % err)
218 typ = None
219 extn = "." + parts[-1] + extn
220 numstring = ""
221 try:
222 stem , numstring, postnum = numstem(".".join(parts[:-1]))
223 except Exception, err:
224 logger.debug("l202: %s" % err)
225 raise
226 if numstring.isdigit():
227 num = int(numstring)
228 ndigit = len(numstring)
229
230
231 self.stem = stem
232 self.num = num
233 self.directory = direc
234 self.format = typ
235 self.extension = extn
236 self.postnum = postnum
237 self.digits = ndigit
238 self.compressed = compressed
239
241 """ cant see how to do without reversing strings
242 Match 1 or more digits going backwards from the end of the string
243 """
244 reg = re.compile(r"^(.*?)(-?[0-9]{0,9})(\D*)$")
245
246 try:
247 res = reg.match(name).groups()
248
249
250 if len(res[0]) == len(res[1]) == 0:
251 return [res[2], '', '']
252 return [ r for r in res]
253 except AttributeError:
254 return [name, "", ""]
255
258 """
259 Function for backward compatibility.
260 Deprecated
261 """
262 return FilenameObject(filename=filename)
263
265 "Try to construct the filename for a given frame"
266 fobj = FilenameObject(filename=filename)
267 if frame is not None:
268 fobj.num = frame
269 return fobj.tostring()
270
272 """ increment number """
273 fobj = FilenameObject(filename=name)
274 fobj.num += 1
275 if not padding:
276 fobj.digits = 0
277 return fobj.tostring()
278
280 """ decrement number """
281 fobj = FilenameObject(filename=name)
282 fobj.num -= 1
283 if not padding:
284 fobj.digits = 0
285 return fobj.tostring()
286
288 """ jump to number """
289 fobj = FilenameObject(filename=name)
290 fobj.num = num
291 if not padding:
292 fobj.digits = 0
293 return fobj.tostring()
294
297 """ extract file number """
298 fobj = FilenameObject(filename=name)
299 return fobj.num
300
301 -def isAscii(name, listExcluded=None):
302 """
303 @param name: string to check
304 @param listExcluded: list of char or string excluded.
305 @return: True of False whether name is pure ascii or not
306 """
307 isascii = None
308 try:
309 name.decode("ascii")
310 except UnicodeDecodeError:
311 isascii = False
312 else:
313 if listExcluded:
314 isascii = not(any(bad in name for bad in listExcluded))
315 else:
316 isascii = True
317 return isascii
318
320 """
321 @param name: string to check
322 @param excluded: tuple of char or string excluded (not list: they are mutable).
323 @return: the name with all non valid char removed
324 """
325 if excluded not in dictAscii:
326 ascii = dictAscii[None][:]
327 for i in excluded:
328 if i in ascii:
329 ascii.remove(i)
330 else:
331 logger.error("toAscii: % not in ascii table" % i)
332 dictAscii[excluded] = ascii
333 else:
334 ascii = dictAscii[excluded]
335 out = [i for i in str(name) if i in ascii]
336 return "".join(out)
337
339 """
340 Workaround that int('1.0') raises an exception
341
342 @param s: string to be converted to integer
343 """
344 try:
345 return int(s)
346 except ValueError:
347 return int(float(s))
348
351 """
352 just an interface providing the name and mode property to a StringIO
353
354 BugFix for MacOSX mainly
355 """
356 - def __init__(self, data, fname=None, mode="r"):
357 stringIO.StringIO.__init__(self, data)
358 self.closed = False
359 if fname == None:
360 self.name = "fabioStream"
361 else:
362 self.name = fname
363 self.mode = mode
364 self.lock = threading.Semaphore()
365 self.__size = None
366
368 if self.__size is None:
369 logger.debug("Measuring size of %s" % self.name)
370 with self.lock:
371 pos = self.tell()
372 self.seek(0, os.SEEK_END)
373 self.__size = self.tell()
374 self.seek(pos)
375 return self.__size
378 size = property(getSize, setSize)
379
381 """
382 wrapper for "file" with locking
383 """
384 - def __init__(self, name, mode="rb", buffering=0):
385 """file(name[, mode[, buffering]]) -> file object
386
387 Open a file. The mode can be 'r', 'w' or 'a' for reading (default),
388 writing or appending. The file will be created if it doesn't exist
389 when opened for writing or appending; it will be truncated when
390 opened for writing. Add a 'b' to the mode for binary files.
391 Add a '+' to the mode to allow simultaneous reading and writing.
392 If the buffering argument is given, 0 means unbuffered, 1 means line
393 buffered, and larger numbers specify the buffer size. The preferred way
394 to open a file is with the builtin open() function.
395 Add a 'U' to mode to open the file for input with universal newline
396 support. Any line ending in the input file will be seen as a '\n'
397 in Python. Also, a file so opened gains the attribute 'newlines';
398 the value for this attribute is one of None (no newline read yet),
399 '\r', '\n', '\r\n' or a tuple containing all the newline types seen.
400
401 'U' cannot be combined with 'w' or '+' mode.
402 """
403 file.__init__(self, name, mode, buffering)
404 self.lock = threading.Semaphore()
405 self.__size = None
407 if self.__size is None:
408 logger.debug("Measuring size of %s" % self.name)
409 with self.lock:
410 pos = self.tell()
411 self.seek(0, os.SEEK_END)
412 self.__size = self.tell()
413 self.seek(pos)
414 return self.__size
417 size = property(getSize, setSize)
418
420 """
421 wrapper for "File" with locking
422 """
423 - def __init__(self, name, mode="rb", buffering=0):
424 logger.warning("No decompressor found for this type of file (are gzip anf bz2 installed ???")
425 File.__init__(self, name, mode, buffering)
426
427 if gzip is None:
428 GzipFile = UnknownCompressedFile
429 else:
431 """
432 Just a wrapper forgzip.GzipFile providing the correct seek capabilities for python 2.5
433 """
434 - def __init__(self, filename=None, mode=None, compresslevel=9, fileobj=None):
435 """
436 Wrapper with locking for constructor for the GzipFile class.
437
438 At least one of fileobj and filename must be given a
439 non-trivial value.
440
441 The new class instance is based on fileobj, which can be a regular
442 file, a StringIO object, or any other object which simulates a file.
443 It defaults to None, in which case filename is opened to provide
444 a file object.
445
446 When fileobj is not None, the filename argument is only used to be
447 included in the gzip file header, which may includes the original
448 filename of the uncompressed file. It defaults to the filename of
449 fileobj, if discernible; otherwise, it defaults to the empty string,
450 and in this case the original filename is not included in the header.
451
452 The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb',
453 depending on whether the file will be read or written. The default
454 is the mode of fileobj if discernible; otherwise, the default is 'rb'.
455 Be aware that only the 'rb', 'ab', and 'wb' values should be used
456 for cross-platform portability.
457
458 The compresslevel argument is an integer from 1 to 9 controlling the
459 level of compression; 1 is fastest and produces the least compression,
460 and 9 is slowest and produces the most compression. The default is 9.
461 """
462 gzip.GzipFile.__init__(self, filename, mode, compresslevel, fileobj)
463 self.lock = threading.Semaphore()
464 self.__size = None
465
466
467 if sys.version_info < (2, 7):
469 if self.__size is None:
470 logger.debug("Measuring size of %s" % self.name)
471 with open(self.filename, "rb") as f:
472 f.seek(-4)
473 self.__size = numpy.fromstring(f.read(4), dtype=numpy.uint32)
474 return self.__size
477 size = property(getSize, setSize)
478 @property
480 return self.fileobj is None
481
482 - def seek(self, offset, whence=os.SEEK_SET):
483 """
484 Move to new file position.
485
486 Argument offset is a byte count. Optional argument whence defaults to
487 0 (offset from start of file, offset should be >= 0); other values are 1
488 (move relative to current position, positive or negative), and 2 (move
489 relative to end of file, usually negative, although many platforms allow
490 seeking beyond the end of a file). If the file is opened in text mode,
491 only offsets returned by tell() are legal. Use of other offsets causes
492 undefined behavior.
493
494 This is a wrapper for seek to ensure compatibility with old python 2.5
495 """
496 if whence == os.SEEK_SET:
497 gzip.GzipFile.seek(self, offset)
498 elif whence == os.SEEK_CUR:
499 gzip.GzipFile.seek(self, offset + self.tell())
500 elif whence == os.SEEK_END:
501 gzip.GzipFile.seek(self, -1)
502 gzip.GzipFile.seek(self, offset + self.tell())
503
504 if bz2 is None:
505 BZ2File = UnknownCompressedFile
506 else:
508 "Wrapper with lock"
509 - def __init__(self, name , mode='r', buffering=0, compresslevel=9):
510 """
511 BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object
512
513 Open a bz2 file. The mode can be 'r' or 'w', for reading (default) or
514 writing. When opened for writing, the file will be created if it doesn't
515 exist, and truncated otherwise. If the buffering argument is given, 0 means
516 unbuffered, and larger numbers specify the buffer size. If compresslevel
517 is given, must be a number between 1 and 9.
518
519 Add a 'U' to mode to open the file for input with universal newline
520 support. Any line ending in the input file will be seen as a '\n' in
521 Python. Also, a file so opened gains the attribute 'newlines'; the value
522 for this attribute is one of None (no newline read yet), '\r', '\n',
523 '\r\n' or a tuple containing all the newline types seen. Universal
524 newlines are available only when reading.
525 """
526 bz2.BZ2File.__init__(self, name , mode, buffering, compresslevel)
527 self.lock = threading.Semaphore()
528 self.__size = None
530 if self.__size is None:
531 logger.debug("Measuring size of %s" % self.name)
532 with self.lock:
533 pos = self.tell()
534 all = self.read()
535 self.__size = self.tell()
536 self.seek(pos)
537 return self.__size
540 size = property(getSize, setSize)
541