Package fabio :: Module fabioutils
[hide private]
[frames] | no frames]

Source Code for Module fabio.fabioutils

  1  #!/usr/bin/env python 
  2  #coding: utf8 
  3   
  4  """ 
  5  General purpose utilities functions for fabio 
  6  """ 
  7  from __future__ import with_statement 
  8  import re, os, logging, threading, sys 
  9  import StringIO as stringIO 
 10  logger = logging.getLogger("fabioutils") 
 11  from compression import bz2, gzip 
 12  import traceback 
 13   
 14   
 15   
 16  FILETYPES = { 
 17      # extension NNNimage fabioclass 
 18      # type consistency - always use a list if one case is 
 19      'edf'    : ['edf'], 
 20      'cor'    : ['edf'], 
 21      'pnm'    : ['pnm'], 
 22      'pgm'    : ['pnm'], 
 23      'pbm'    : ['pnm'], 
 24      'tif'    : ['tif'], 
 25      'tiff'   : ['tif'], 
 26      'img'    : ['adsc', 'OXD', 'HiPiC'], 
 27      'mccd'   : ['marccd'], 
 28      'mar2300': ['mar345'], 
 29      'sfrm'   : ['bruker100'], 
 30      'msk'    : ['fit2dmask'], 
 31      'spr'    : ['fit2dspreadsheet'], 
 32      'dm3'    : ['dm3'], 
 33      'kcd'    : ['kcd'], 
 34      'cbf'    : ['cbf'], 
 35      'xml'    : ["xsd"], 
 36      'xsd'    : ["xsd"], 
 37               } 
 38   
 39  # Add bzipped and gzipped 
 40  for key in FILETYPES.keys(): 
 41      FILETYPES[key + ".bz2"] = FILETYPES[key] 
 42      FILETYPES[key + ".gz"] = FILETYPES[key] 
 43   
 44   
 45  # Compressors 
 46   
 47  COMPRESSORS = {} 
 48   
 49   
 50  dictAscii = {None:[chr(i) for i in range(32, 127)], 
 51             } 
 52   
 53  try: 
 54      lines = os.popen("gzip -h 2>&1").read() 
 55      # Looking for "usage" 
 56      if "sage" in lines: 
 57          COMPRESSORS['.gz'] = 'gzip -dc ' 
 58      else: 
 59          COMPRESSORS['.gz'] = None 
 60  except Exception: 
 61      COMPRESSORS['.gz'] = None 
 62   
 63  try: 
 64      lines = os.popen("bzip2 -h 2>&1").read() 
 65      # Looking for "usage"  
 66      if "sage" in lines: 
 67          COMPRESSORS['.bz2'] = 'bzip2 -dc ' 
 68      else: 
 69          COMPRESSORS['.bz2'] = None 
 70  except Exception: 
 71      COMPRESSORS['.bz2'] = None 
72 73 -def deprecated(func):
74 """ 75 used to deprecate a function/method: prints a lot of warning messages to enforce the modifaction of the code 76 """ 77 def wrapper(*arg, **kw): 78 """ 79 decorator that deprecates the use of a function 80 """ 81 logger.warning("%s is Deprecated !!! %s" % (func.func_name, os.linesep.join([""] + traceback.format_stack()[:-1]))) 82 return func(*arg, **kw)
83 return wrapper 84
85 86 -def getnum(name):
87 """ 88 # try to figure out a file number 89 # guess it starts at the back 90 """ 91 stem , num, post_num = numstem(name) 92 try: 93 return int(num) 94 except ValueError: 95 return None
96
97 -class FilenameObject(object):
98 """ 99 The 'meaning' of a filename ... 100 """
101 - def __init__(self, stem=None, 102 num=None, 103 directory=None, 104 format=None, 105 extension=None, 106 postnum=None, 107 digits=4, 108 filename = None):
109 """ 110 This class can either be instanciated by a set of parameters like directory, prefix, num, extension, ... 111 112 @param stem: the stem is a kind of prefix (str) 113 @param num: image number in the serie (int) 114 @param directory: name of the directory (str) 115 @param format: ?? 116 @param extension: 117 @param postnum: 118 @param digits: Number of digits used to print num 119 120 Alternative constructor: 121 122 @param filename: fullpath of an image file to be deconstructed into directory, prefix, num, extension, ... 123 124 """ 125 126 127 self.stem = stem 128 self.num = num 129 self.format = format 130 self.extension = extension 131 self.digits = digits 132 self.postnum = postnum 133 self.directory = directory 134 self.compressed = None 135 if filename is not None: 136 self.deconstruct_filename(filename)
137 138
139 - def str(self):
140 """ Return a string representation """ 141 fmt = "stem %s, num %s format %s extension %s " + \ 142 "postnum = %s digits %s dir %s" 143 return fmt % tuple([str(x) for x in [ 144 self.stem , 145 self.num , 146 self.format , 147 self.extension , 148 self.postnum , 149 self.digits , 150 self.directory ] ])
151 __repr__ = str 152
153 - def tostring(self):
154 """ 155 convert yourself to a string 156 """ 157 name = self.stem 158 if self.digits is not None and self.num is not None: 159 fmt = "%0" + str(self.digits) + "d" 160 name += fmt % self.num 161 if self.postnum is not None: 162 name += self.postnum 163 if self.extension is not None: 164 name += self.extension 165 if self.directory is not None: 166 name = os.path.join(self.directory, name) 167 return name
168 169
170 - def deconstruct_filename(self, filename):
171 """ 172 Break up a filename to get image type and number 173 """ 174 direc, name = os.path.split(filename) 175 direc = direc or None 176 parts = name.split(".") 177 compressed = False 178 stem = parts[0] 179 extn = "" 180 postnum = "" 181 ndigit = 4 182 num = None 183 typ = None 184 if parts[-1] in ["gz", "bz2"]: 185 extn = "." + parts[-1] 186 parts = parts[:-1] 187 compressed = True 188 if parts[-1] in FILETYPES.keys(): 189 typ = FILETYPES[parts[-1]] 190 extn = "." + parts[-1] + extn 191 try: 192 stem, numstring, postnum = numstem(".".join(parts[:-1])) 193 num = int(numstring) 194 ndigit = len(numstring) 195 except Exception, err: 196 # There is no number - hence make num be None, not 0 197 logger.debug("l176: %s" % err) 198 num = None 199 stem = "".join(parts[:-1]) 200 else: 201 # Probably two type left 202 if len(parts) == 1: 203 # Probably GE format stem_numb 204 parts2 = parts[0].split("_") 205 if parts2[-1].isdigit(): 206 num = int(parts2[-1]) 207 ndigit = len(parts2[-1]) 208 typ = ['GE'] 209 stem = "_".join(parts2[:-1]) + "_" 210 else: 211 try: 212 num = int(parts[-1]) 213 ndigit = len(parts[-1]) 214 typ = ['bruker'] 215 stem = ".".join(parts[:-1]) + "." 216 except Exception, err: 217 logger.debug("l196: %s" % err) 218 typ = None 219 extn = "." + parts[-1] + extn 220 numstring = "" 221 try: 222 stem , numstring, postnum = numstem(".".join(parts[:-1])) 223 except Exception, err: 224 logger.debug("l202: %s" % err) 225 raise 226 if numstring.isdigit(): 227 num = int(numstring) 228 ndigit = len(numstring) 229 # raise Exception("Cannot decode "+filename) 230 231 self.stem = stem 232 self.num = num 233 self.directory = direc 234 self.format = typ 235 self.extension = extn 236 self.postnum = postnum 237 self.digits = ndigit 238 self.compressed = compressed
239
240 -def numstem(name):
241 """ cant see how to do without reversing strings 242 Match 1 or more digits going backwards from the end of the string 243 """ 244 reg = re.compile(r"^(.*?)(-?[0-9]{0,9})(\D*)$") 245 #reg = re.compile("""(\D*)(\d\d*)(\w*)""") 246 try: 247 res = reg.match(name).groups() 248 #res = reg.match(name[::-1]).groups() 249 #return [ r[::-1] for r in res[::-1]] 250 if len(res[0]) == len(res[1]) == 0: # Hack for file without number 251 return [res[2], '', ''] 252 return [ r for r in res] 253 except AttributeError: # no digits found 254 return [name, "", ""]
255
256 @deprecated 257 -def deconstruct_filename(filename):
258 """ 259 Function for backward compatibility. 260 Deprecated 261 """ 262 return FilenameObject(filename=filename)
263
264 -def construct_filename(filename, frame=None):
265 "Try to construct the filename for a given frame" 266 fobj = FilenameObject(filename=filename) 267 if frame is not None: 268 fobj.num = frame 269 return fobj.tostring()
270
271 -def next_filename(name, padding=True):
272 """ increment number """ 273 fobj = FilenameObject(filename=name) 274 fobj.num += 1 275 if not padding: 276 fobj.digits = 0 277 return fobj.tostring()
278
279 -def previous_filename(name, padding=True):
280 """ decrement number """ 281 fobj = FilenameObject(filename=name) 282 fobj.num -= 1 283 if not padding: 284 fobj.digits = 0 285 return fobj.tostring()
286
287 -def jump_filename(name, num, padding=True):
288 """ jump to number """ 289 fobj = FilenameObject(filename=name) 290 fobj.num = num 291 if not padding: 292 fobj.digits = 0 293 return fobj.tostring()
294
295 296 -def extract_filenumber(name):
297 """ extract file number """ 298 fobj = FilenameObject(filename=name) 299 return fobj.num
300
301 -def isAscii(name, listExcluded=None):
302 """ 303 @param name: string to check 304 @param listExcluded: list of char or string excluded. 305 @return: True of False whether name is pure ascii or not 306 """ 307 isascii = None 308 try: 309 name.decode("ascii") 310 except UnicodeDecodeError: 311 isascii = False 312 else: 313 if listExcluded: 314 isascii = not(any(bad in name for bad in listExcluded)) 315 else: 316 isascii = True 317 return isascii
318
319 -def toAscii(name, excluded=None):
320 """ 321 @param name: string to check 322 @param excluded: tuple of char or string excluded (not list: they are mutable). 323 @return: the name with all non valid char removed 324 """ 325 if excluded not in dictAscii: 326 ascii = dictAscii[None][:] 327 for i in excluded: 328 if i in ascii: 329 ascii.remove(i) 330 else: 331 logger.error("toAscii: % not in ascii table" % i) 332 dictAscii[excluded] = ascii 333 else: 334 ascii = dictAscii[excluded] 335 out = [i for i in str(name) if i in ascii] 336 return "".join(out)
337
338 -def nice_int(s):
339 """ 340 Workaround that int('1.0') raises an exception 341 342 @param s: string to be converted to integer 343 """ 344 try: 345 return int(s) 346 except ValueError: 347 return int(float(s))
348
349 350 -class StringIO(stringIO.StringIO):
351 """ 352 just an interface providing the name and mode property to a StringIO 353 354 BugFix for MacOSX mainly 355 """
356 - def __init__(self, data, fname=None, mode="r"):
357 stringIO.StringIO.__init__(self, data) 358 self.closed = False 359 if fname == None: 360 self.name = "fabioStream" 361 else: 362 self.name = fname 363 self.mode = mode 364 self.lock = threading.Semaphore() 365 self.__size = None
366
367 - def getSize(self):
368 if self.__size is None: 369 logger.debug("Measuring size of %s" % self.name) 370 with self.lock: 371 pos = self.tell() 372 self.seek(0, os.SEEK_END) 373 self.__size = self.tell() 374 self.seek(pos) 375 return self.__size
376 - def setSize(self, size):
377 self.__size = size
378 size = property(getSize, setSize)
379
380 -class File(file):
381 """ 382 wrapper for "file" with locking 383 """
384 - def __init__(self, name, mode="rb", buffering=0):
385 """file(name[, mode[, buffering]]) -> file object 386 387 Open a file. The mode can be 'r', 'w' or 'a' for reading (default), 388 writing or appending. The file will be created if it doesn't exist 389 when opened for writing or appending; it will be truncated when 390 opened for writing. Add a 'b' to the mode for binary files. 391 Add a '+' to the mode to allow simultaneous reading and writing. 392 If the buffering argument is given, 0 means unbuffered, 1 means line 393 buffered, and larger numbers specify the buffer size. The preferred way 394 to open a file is with the builtin open() function. 395 Add a 'U' to mode to open the file for input with universal newline 396 support. Any line ending in the input file will be seen as a '\n' 397 in Python. Also, a file so opened gains the attribute 'newlines'; 398 the value for this attribute is one of None (no newline read yet), 399 '\r', '\n', '\r\n' or a tuple containing all the newline types seen. 400 401 'U' cannot be combined with 'w' or '+' mode. 402 """ 403 file.__init__(self, name, mode, buffering) 404 self.lock = threading.Semaphore() 405 self.__size = None
406 - def getSize(self):
407 if self.__size is None: 408 logger.debug("Measuring size of %s" % self.name) 409 with self.lock: 410 pos = self.tell() 411 self.seek(0, os.SEEK_END) 412 self.__size = self.tell() 413 self.seek(pos) 414 return self.__size
415 - def setSize(self, size):
416 self.__size = size
417 size = property(getSize, setSize)
418
419 -class UnknownCompressedFile(File):
420 """ 421 wrapper for "File" with locking 422 """
423 - def __init__(self, name, mode="rb", buffering=0):
424 logger.warning("No decompressor found for this type of file (are gzip anf bz2 installed ???") 425 File.__init__(self, name, mode, buffering)
426 427 if gzip is None: 428 GzipFile = UnknownCompressedFile 429 else:
430 - class GzipFile(gzip.GzipFile):
431 """ 432 Just a wrapper forgzip.GzipFile providing the correct seek capabilities for python 2.5 433 """
434 - def __init__(self, filename=None, mode=None, compresslevel=9, fileobj=None):
435 """ 436 Wrapper with locking for constructor for the GzipFile class. 437 438 At least one of fileobj and filename must be given a 439 non-trivial value. 440 441 The new class instance is based on fileobj, which can be a regular 442 file, a StringIO object, or any other object which simulates a file. 443 It defaults to None, in which case filename is opened to provide 444 a file object. 445 446 When fileobj is not None, the filename argument is only used to be 447 included in the gzip file header, which may includes the original 448 filename of the uncompressed file. It defaults to the filename of 449 fileobj, if discernible; otherwise, it defaults to the empty string, 450 and in this case the original filename is not included in the header. 451 452 The mode argument can be any of 'r', 'rb', 'a', 'ab', 'w', or 'wb', 453 depending on whether the file will be read or written. The default 454 is the mode of fileobj if discernible; otherwise, the default is 'rb'. 455 Be aware that only the 'rb', 'ab', and 'wb' values should be used 456 for cross-platform portability. 457 458 The compresslevel argument is an integer from 1 to 9 controlling the 459 level of compression; 1 is fastest and produces the least compression, 460 and 9 is slowest and produces the most compression. The default is 9. 461 """ 462 gzip.GzipFile.__init__(self, filename, mode, compresslevel, fileobj) 463 self.lock = threading.Semaphore() 464 self.__size = None
465 466 467 if sys.version_info < (2, 7):
468 - def getSize(self):
469 if self.__size is None: 470 logger.debug("Measuring size of %s" % self.name) 471 with open(self.filename, "rb") as f: 472 f.seek(-4) 473 self.__size = numpy.fromstring(f.read(4), dtype=numpy.uint32) 474 return self.__size
475 - def setSize(self, value):
476 self.__size = value
477 size = property(getSize, setSize) 478 @property
479 - def closed(self):
480 return self.fileobj is None
481
482 - def seek(self, offset, whence=os.SEEK_SET):
483 """ 484 Move to new file position. 485 486 Argument offset is a byte count. Optional argument whence defaults to 487 0 (offset from start of file, offset should be >= 0); other values are 1 488 (move relative to current position, positive or negative), and 2 (move 489 relative to end of file, usually negative, although many platforms allow 490 seeking beyond the end of a file). If the file is opened in text mode, 491 only offsets returned by tell() are legal. Use of other offsets causes 492 undefined behavior. 493 494 This is a wrapper for seek to ensure compatibility with old python 2.5 495 """ 496 if whence == os.SEEK_SET: 497 gzip.GzipFile.seek(self, offset) 498 elif whence == os.SEEK_CUR: 499 gzip.GzipFile.seek(self, offset + self.tell()) 500 elif whence == os.SEEK_END: 501 gzip.GzipFile.seek(self, -1) 502 gzip.GzipFile.seek(self, offset + self.tell())
503 504 if bz2 is None: 505 BZ2File = UnknownCompressedFile 506 else:
507 - class BZ2File(bz2.BZ2File):
508 "Wrapper with lock"
509 - def __init__(self, name , mode='r', buffering=0, compresslevel=9):
510 """ 511 BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object 512 513 Open a bz2 file. The mode can be 'r' or 'w', for reading (default) or 514 writing. When opened for writing, the file will be created if it doesn't 515 exist, and truncated otherwise. If the buffering argument is given, 0 means 516 unbuffered, and larger numbers specify the buffer size. If compresslevel 517 is given, must be a number between 1 and 9. 518 519 Add a 'U' to mode to open the file for input with universal newline 520 support. Any line ending in the input file will be seen as a '\n' in 521 Python. Also, a file so opened gains the attribute 'newlines'; the value 522 for this attribute is one of None (no newline read yet), '\r', '\n', 523 '\r\n' or a tuple containing all the newline types seen. Universal 524 newlines are available only when reading. 525 """ 526 bz2.BZ2File.__init__(self, name , mode, buffering, compresslevel) 527 self.lock = threading.Semaphore() 528 self.__size = None
529 - def getSize(self):
530 if self.__size is None: 531 logger.debug("Measuring size of %s" % self.name) 532 with self.lock: 533 pos = self.tell() 534 all = self.read() 535 self.__size = self.tell() 536 self.seek(pos) 537 return self.__size
538 - def setSize(self, value):
539 self.__size = value
540 size = property(getSize, setSize)
541