1
2
3 """
4 Authors: Jérôme Kieffer, ESRF
5 email:jerome.kieffer@esrf.fr
6
7 Cif Binary Files images are 2D images written by the Pilatus detector and others.
8 They use a modified (simplified) byte-offset algorithm.
9
10 CIF is a library for manipulating Crystallographic information files and tries
11 to conform to the specification of the IUCR
12 """
13 __author__ = "Jérôme Kieffer"
14 __contact__ = "jerome.kieffer@esrf.eu"
15 __license__ = "GPLv3+"
16 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
17 __version__ = ["Generated by CIF.py: Jan 2005 - April 2012",
18 "Written by Jerome Kieffer: Jerome.Kieffer@esrf.eu",
19 "On-line data analysis / ISDD ", "ESRF Grenoble (France)"]
20
21
22 import os, logging
23 logger = logging.getLogger("cbfimage")
24 import numpy
25 from fabioimage import fabioimage
26 from compression import decByteOffet_numpy, md5sum, compByteOffet_numpy
27
28
29 DATA_TYPES = { "signed 8-bit integer" : numpy.int8,
30 "signed 16-bit integer" : numpy.int16,
31 "signed 32-bit integer" : numpy.int32,
32 "signed 64-bit integer" : numpy.int64
33 }
34
35 MINIMUM_KEYS = ["X-Binary-Size-Fastest-Dimension",
36 'ByteOrder',
37 'Data type',
38 'X dimension',
39 'Y dimension',
40 'Number of readouts']
41
42
43 STARTER = "\x0c\x1a\x04\xd5"
44 PADDING = 512
47 """
48 Read the Cif Binary File data format
49 """
50 - def __init__(self, data=None , header=None, fname=None):
51 """
52 Constructor of the class CIF Binary File reader.
53
54 @param _strFilename: the name of the file to open
55 @type _strFilename: string
56 """
57 fabioimage.__init__(self, data, header)
58 self.cif = CIF()
59 if fname is not None:
60 self.read(fname)
61
62 @staticmethod
64 if data is None:
65 return None
66 elif numpy.issubdtype(data.dtype, int):
67 return data
68 else:
69 return data.astype(int)
70
71
73 """
74 Read in a header in some CBF format from a string representing binary stuff
75
76 @param inStream: file containing the Cif Binary part.
77 @type inStream: opened file.
78 """
79 self.cif.loadCIF(inStream, _bKeepComment=True)
80
81
82 for key in self.cif:
83 if key != "_array_data.data":
84 self.header_keys.append(key)
85 self.header[key] = self.cif[key].strip(" \"\n\r\t")
86
87 if not "_array_data.data" in self.cif:
88 raise Exception("cbfimage: CBF file %s is corrupt, cannot find data block with '_array_data.data' key" % self.fname)
89
90 inStream2 = self.cif["_array_data.data"]
91 sep = "\r\n"
92 iSepPos = inStream2.find(sep)
93 if iSepPos < 0 or iSepPos > 80:
94 sep = "\n"
95
96 lines = inStream2.split(sep)
97 for oneLine in lines[1:]:
98 if len(oneLine) < 10:
99 break
100 try:
101 key, val = oneLine.split(':' , 1)
102 except ValueError:
103 key, val = oneLine.split('=' , 1)
104 key = key.strip()
105 self.header_keys.append(key)
106 self.header[key] = val.strip(" \"\n\r\t")
107 missing = []
108 for item in MINIMUM_KEYS:
109 if item not in self.header_keys:
110 missing.append(item)
111 if len(missing) > 0:
112 logger.debug("CBF file misses the keys " + " ".join(missing))
113
114
115 - def read(self, fname, frame=None):
116 """
117 Read in header into self.header and
118 the data into self.data
119 """
120 self.filename = fname
121 self.header = {}
122 self.resetvals()
123
124 infile = self._open(fname, "rb")
125 self._readheader(infile)
126
127 try:
128 self.dim1 = int(self.header['X-Binary-Size-Fastest-Dimension'])
129 self.dim2 = int(self.header['X-Binary-Size-Second-Dimension'])
130 except:
131 raise Exception(IOError, "CBF file %s is corrupt, no dimensions in it" % fname)
132 try:
133 bytecode = DATA_TYPES[self.header['X-Binary-Element-Type']]
134 self.bpp = len(numpy.array(0, bytecode).tostring())
135 except KeyError:
136 bytecode = numpy.int32
137 self.bpp = 32
138 logger.warning("Defaulting type to int32")
139 if self.header["conversions"] == "x-CBF_BYTE_OFFSET":
140 self.data = self._readbinary_byte_offset(self.cif["_array_data.data"]).astype(bytecode).reshape((self.dim2, self.dim1))
141 else:
142 raise Exception(IOError, "Compression scheme not yet supported, please contact FABIO development team")
143
144 self.bytecode = self.data.dtype.type
145 self.resetvals()
146
147 self.pilimage = None
148 return self
149
150
151
153 """
154 Read in a binary part of an x-CBF_BYTE_OFFSET compressed image
155
156 @param inStream: the binary image (without any CIF decorators)
157 @type inStream: python string.
158 @return: a linear numpy array without shape and dtype set
159 @rtype: numpy array
160 """
161 startPos = inStream.find(STARTER) + 4
162 data = inStream[ startPos: startPos + int(self.header["X-Binary-Size"])]
163 try:
164 import byte_offset
165 except ImportError:
166 logger.warning("Error in byte_offset part: Falling back to Numpy implementation")
167 myData = decByteOffet_numpy(data, size=self.dim1 * self.dim2)
168 else:
169 myData = byte_offset.analyseCython(data, size=self.dim1 * self.dim2)
170
171 assert len(myData) == self.dim1 * self.dim2
172 return myData
173
174
176 """
177 write the file in CBF format
178 @param fname: name of the file
179 @type: string
180 """
181 if self.data is not None:
182 self.dim2, self.dim1 = self.data.shape
183 else:
184 raise RuntimeError("CBF image contains no data")
185 binary_blob = compByteOffet_numpy(self.data)
186
187
188
189
190 dtype = "Unknown"
191 for key, value in DATA_TYPES.iteritems():
192 if value == self.data.dtype:
193 dtype = key
194 binary_block = [
195 "--CIF-BINARY-FORMAT-SECTION--",
196 "Content-Type: application/octet-stream;",
197 ' conversions="x-CBF_BYTE_OFFSET"',
198 'Content-Transfer-Encoding: BINARY',
199 "X-Binary-Size: %d" % (len(binary_blob)),
200 "X-Binary-ID: 1",
201 'X-Binary-Element-Type: "%s"' % (dtype),
202 "X-Binary-Element-Byte-Order: LITTLE_ENDIAN" ,
203 "Content-MD5: %s" % md5sum(binary_blob),
204 "X-Binary-Number-of-Elements: %s" % (self.dim1 * self.dim2),
205 "X-Binary-Size-Fastest-Dimension: %d" % self.dim1,
206 "X-Binary-Size-Second-Dimension: %d" % self.dim2,
207 "X-Binary-Size-Padding: %d" % 1,
208 "",
209 STARTER + binary_blob,
210 "",
211 "--CIF-BINARY-FORMAT-SECTION----"
212 ]
213
214 if "_array_data.header_contents" not in self.header:
215 nonCifHeaders = []
216 else:
217 nonCifHeaders = [i.strip()[2:] for i in self.header["_array_data.header_contents"].split("\n") if i.find("# ") >= 0]
218
219 for key in self.header:
220 if (key not in self.header_keys):
221 self.header_keys.append(key)
222 for key in self.header_keys:
223 if key.startswith("_") :
224 if key not in self.cif or self.cif[key] != self.header[key]:
225 self.cif[key] = self.header[key]
226 elif key.startswith("X-Binary-"):
227 pass
228 elif key.startswith("Content-"):
229 pass
230 elif key.startswith("conversions"):
231 pass
232 elif key.startswith("filename"):
233 pass
234 elif key in self.header:
235 nonCifHeaders.append("%s %s" % (key, self.header[key]))
236 if len(nonCifHeaders) > 0:
237 self.cif["_array_data.header_contents"] = "\r\n".join(["# %s" % i for i in nonCifHeaders])
238
239 self.cif["_array_data.data"] = "\r\n".join(binary_block)
240 self.cif.saveCIF(fname, linesep="\r\n", binary=True)
241
242
243
244
245 -class CIF(dict):
246 """
247 This is the CIF class, it represents the CIF dictionary;
248 and as a a python dictionary thus inherits from the dict built in class.
249 """
250 EOL = ["\r", "\n", "\r\n", "\n\r"]
251 BLANK = [" ", "\t"] + EOL
252 START_COMMENT = ["\"", "\'"]
253 BINARY_MARKER = "--CIF-BINARY-FORMAT-SECTION--"
254
256 """
257 Constructor of the class.
258
259 @param _strFilename: the name of the file to open
260 @type _strFilename: filename (str) or file object
261 """
262 dict.__init__(self)
263 self._ordered = []
264 if _strFilename is not None:
265 self.loadCIF(_strFilename)
266
268 if key not in self._ordered:
269 self._ordered.append(key)
270 return dict.__setitem__(self, key, value)
271
272 - def pop(self, key):
273 if key in self._ordered:
274 self._ordered.remove(key)
275 return dict.pop(self, key)
276
278 if key in self._ordered:
279 self._ordered.remove(key)
280 return dict.popitem(self, key)
281
282
283 - def loadCIF(self, _strFilename, _bKeepComment=False):
284 """Load the CIF file and populates the CIF dictionary into the object
285 @param _strFilename: the name of the file to open
286 @type _strFilename: string
287 @param _strFilename: the name of the file to open
288 @type _strFilename: string
289 @return: None
290 """
291
292 if isinstance(_strFilename, (str, unicode)):
293 if os.path.isfile(_strFilename):
294 infile = open(_strFilename, "rb")
295 else:
296 raise RuntimeError("CIF.loadCIF: No such file to open: %s" % _strFilename)
297
298 elif "read" in dir(_strFilename):
299 infile = _strFilename
300 else:
301 raise RuntimeError("CIF.loadCIF: what is %s type %s" % (_strFilename, type(_strFilename)))
302 if _bKeepComment:
303 self._parseCIF(infile.read())
304 else:
305 self._parseCIF(CIF._readCIF(infile))
306 readCIF = loadCIF
307
308 @staticmethod
310 """
311 Check if all characters in a string are ascii,
312
313 @param _strIn: input string
314 @type _strIn: python string
315 @return: boolean
316 @rtype: boolean
317 """
318 bIsAcii = True
319 for i in _strIn:
320 if ord(i) > 127:
321 bIsAcii = False
322 break
323 return bIsAcii
324
325
326 @staticmethod
328 """
329 - Check if the filename containing the CIF data exists
330 - read the cif file
331 - removes the comments
332
333 @param _instream: the file containing the CIF data
334 @type _instream: open file in read mode
335 @return: a string containing the raw data
336 @rtype: string
337 """
338 if not "readlines" in dir(_instream):
339 raise RuntimeError("CIF._readCIF(instream): I expected instream to be an opened file,\
340 here I got %s type %s" % (_instream, type(_instream)))
341 lLinesRead = _instream.readlines()
342 sText = ""
343 for sLine in lLinesRead:
344 iPos = sLine.find("#")
345 if iPos >= 0:
346 if CIF.isAscii(sLine):
347 sText += sLine[:iPos] + os.linesep
348
349 if iPos > 80 :
350 logger.warning("This line is too long and could cause problems in PreQuest: %s", sLine)
351 else :
352 sText += sLine
353 if len(sLine.strip()) > 80 :
354 logger.warning("This line is too long and could cause problems in PreQues: %s", sLine)
355 return sText
356
357
359 """
360 - Parses the text of a CIF file
361 - Cut it in fields
362 - Find all the loops and process
363 - Find all the keys and values
364
365 @param sText: the content of the CIF - file
366 @type sText: string
367 @return: Nothing, the data are incorporated at the CIF object dictionary
368 @rtype: None
369 """
370 loopidx = []
371 looplen = []
372 loop = []
373
374 lFields = CIF._splitCIF(sText.strip())
375
376 for i in range(len(lFields)):
377 if lFields[i].lower() == "loop_":
378 loopidx.append(i)
379 if len(loopidx) > 0:
380 for i in loopidx:
381 loopone, length, keys = CIF._analyseOneLoop(lFields, i)
382 loop.append([keys, loopone])
383 looplen.append(length)
384
385
386 for i in range(len(loopidx) - 1, -1, -1):
387 f1 = lFields[:loopidx[i]] + lFields[loopidx[i] + looplen[i]:]
388 lFields = f1
389
390 self["loop_"] = loop
391
392 for i in range(len(lFields) - 1):
393
394 if len(lFields[i + 1]) == 0 : lFields[i + 1] = "?"
395 if lFields[i][0] == "_" and lFields[i + 1][0] != "_":
396 self[lFields[i]] = lFields[i + 1]
397
398 @staticmethod
400 """
401 Separate the text in fields as defined in the CIF
402
403 @param sText: the content of the CIF - file
404 @type sText: string
405 @return: list of all the fields of the CIF
406 @rtype: list
407 """
408 lFields = []
409 while True:
410 if len(sText) == 0:
411 break
412 elif sText[0] == "'":
413 idx = 0
414 bFinished = False
415 while not bFinished:
416 idx += 1 + sText[idx + 1:].find("'")
417
418 if idx >= len(sText) - 1:
419
420 lFields.append(sText[1:-1].strip())
421 sText = ""
422 bFinished = True
423 break
424
425 if sText[idx + 1] in CIF.BLANK:
426 lFields.append(sText[1:idx].strip())
427 sText1 = sText[idx + 1:]
428 sText = sText1.strip()
429 bFinished = True
430
431 elif sText[0] == '"':
432 idx = 0
433 bFinished = False
434 while not bFinished:
435 idx += 1 + sText[idx + 1:].find('"')
436
437 if idx >= len(sText) - 1:
438
439 lFields.append(sText[1:-1].strip())
440
441 sText = ""
442 bFinished = True
443 break
444
445 if sText[idx + 1] in CIF.BLANK:
446 lFields.append(sText[1:idx].strip())
447
448 sText1 = sText[idx + 1:]
449 sText = sText1.strip()
450 bFinished = True
451 elif sText[0] == ';':
452 if sText[1:].strip().find(CIF.BINARY_MARKER) == 0:
453 idx = sText[32:].find(CIF.BINARY_MARKER)
454 if idx == -1:
455 idx = 0
456 else:
457 idx += 32 + len(CIF.BINARY_MARKER)
458 else:
459 idx = 0
460 bFinished = False
461 while not bFinished:
462 idx += 1 + sText[idx + 1:].find(';')
463 if sText[idx - 1] in CIF.EOL:
464 lFields.append(sText[1:idx - 1].strip())
465 sText1 = sText[idx + 1:]
466 sText = sText1.strip()
467 bFinished = True
468 else:
469 f = sText.split(None, 1)[0]
470 lFields.append(f)
471
472 sText1 = sText[len(f):].strip()
473 sText = sText1
474 return lFields
475
476
477 @staticmethod
479 """Processes one loop in the data extraction of the CIF file
480 @param lFields: list of all the words contained in the cif file
481 @type lFields: list
482 @param iStart: the starting index corresponding to the "loop_" key
483 @type iStart: integer
484 @return: the list of loop dictionaries, the length of the data
485 extracted from the lFields and the list of all the keys of the loop.
486 @rtype: tuple
487 """
488
489
490
491 loop = []
492 keys = []
493 i = iStart + 1
494 bFinished = False
495 while not bFinished:
496 if lFields[i][0] == "_":
497 keys.append(lFields[i])
498 i += 1
499 else:
500 bFinished = True
501 data = []
502 while True:
503 if i >= len(lFields):
504 break
505 elif len(lFields[i]) == 0:
506 break
507 elif lFields[i][0] == "_":
508 break
509 elif lFields[i] in ["loop_", "stop_", "global_", "data_", "save_"]:
510 break
511 else:
512 data.append(lFields[i])
513 i += 1
514
515 k = 0
516
517 if len(data) < len(keys):
518 element = {}
519 for j in keys:
520 if k < len(data):
521 element[j] = data[k]
522 else :
523 element[j] = "?"
524 k += 1
525
526 loop.append(element)
527
528 else:
529
530
531 for i in range(len(data) / len(keys)):
532 element = {}
533 for j in keys:
534 element[j] = data[k]
535 k += 1
536
537 loop.append(element)
538
539 return loop, 1 + len(keys) + len(data), keys
540
541
542
543
544
545
546
547
548
549
550 - def saveCIF(self, _strFilename="test.cif", linesep=os.linesep, binary=False):
551 """Transforms the CIF object in string then write it into the given file
552 @param _strFilename: the of the file to be written
553 @param linesep: line separation used (to force compatibility with windows/unix)
554 @param binary: Shall we write the data as binary (True only for imageCIF/CBF)
555 @type param: string
556 """
557 if binary:
558 mode = "wb"
559 else:
560 mode = "w"
561 try:
562 fFile = open(_strFilename, mode)
563 except IOError:
564 print("Error during the opening of file for write: %s" %
565 _strFilename)
566 return
567 fFile.write(self.tostring(_strFilename, linesep))
568 try:
569 fFile.close()
570 except IOError:
571 print("Error during the closing of file for write: %s" %
572 _strFilename)
573
574
575 - def tostring(self, _strFilename=None, linesep=os.linesep):
576 """
577 Converts a cif dictionnary to a string according to the CIF syntax
578
579 @param _strFilename: the name of the filename to be appended in the header of the CIF file
580 @type _strFilename: string
581 @return: a sting that corresponds to the content of the CIF - file.
582
583 """
584
585 lstStrCif = ["# " + i for i in __version__]
586 if "_chemical_name_common" in self:
587 t = self["_chemical_name_common"].split()[0]
588 elif _strFilename is not None:
589 t = os.path.splitext(os.path.split(str(_strFilename).strip())[1])[0]
590 else:
591 t = ""
592 lstStrCif.append("data_%s" % (t))
593
594 lKeys = self.keys()
595 lKeys.sort()
596 for key in lKeys[:]:
597 if key in self._ordered:
598 lKeys.remove(key)
599 self._ordered += lKeys
600
601 for sKey in self._ordered:
602 if sKey == "loop_":
603 continue
604 if sKey not in self:
605 self._ordered.remove(sKey)
606 logger.debug("Skipping key %s from ordered list as no more present in dict")
607 continue
608 sValue = str(self[sKey])
609 if sValue.find("\n") > -1:
610 lLine = [sKey, ";", sValue, ";", ""]
611 elif len(sValue.split()) > 1:
612 sLine = "%s '%s'" % (sKey, sValue)
613 if len(sLine) > 80:
614 lLine = [str(sKey), sValue]
615 else:
616 lLine = [sLine]
617 else:
618 sLine = "%s %s" % (sKey, sValue)
619 if len(sLine) > 80:
620 lLine = [str(sKey), sValue]
621 else:
622 lLine = [sLine]
623 lstStrCif += lLine
624 if self.has_key("loop_"):
625 for loop in self["loop_"]:
626 lstStrCif.append("loop_ ")
627 lKeys = loop[0]
628 llData = loop[1]
629 lstStrCif += [" %s" % (sKey) for sKey in lKeys]
630 for lData in llData:
631 sLine = " "
632 for key in lKeys:
633 sRawValue = lData[key]
634 if sRawValue.find("\n") > -1:
635 lstStrCif += [sLine, ";", str(sRawValue), ";"]
636 sLine = " "
637 else:
638 if len(sRawValue.split()) > 1:
639 value = "'%s'" % (sRawValue)
640 else:
641 value = str(sRawValue)
642 if len(sLine) + len(value) > 78:
643 lstStrCif += [sLine]
644 sLine = " " + value
645 else:
646 sLine += " " + value
647 lstStrCif.append(sLine)
648 lstStrCif.append("")
649 return linesep.join(lstStrCif)
650
651
653 """
654 Check if the key exists in the CIF and is non empty.
655 @param sKey: CIF key
656 @type sKey: string
657 @param cif: CIF dictionary
658 @return: True if the key exists in the CIF dictionary and is non empty
659 @rtype: boolean
660 """
661 bExists = False
662 if self.has_key(sKey):
663 if len(self[sKey]) >= 1:
664 if self[sKey][0] not in ["?", "."]:
665 bExists = True
666 return bExists
667
668
670 """
671 Check if the key exists in the CIF dictionary.
672 @param sKey: CIF key
673 @type sKey: string
674 @param cif: CIF dictionary
675 @return: True if the key exists in the CIF dictionary and is non empty
676 @rtype: boolean
677 """
678 if not self.exists("loop_"):
679 return False
680 bExists = False
681 if not bExists:
682 for i in self["loop_"]:
683 for j in i[0]:
684 if j == sKey:
685 bExists = True
686 return bExists
687
688
690 """
691 Load the powder diffraction CHIPLOT file and returns the
692 pd_CIF dictionary in the object
693
694 @param _strFilename: the name of the file to open
695 @type _strFilename: string
696 @return: the CIF object corresponding to the powder diffraction
697 @rtype: dictionary
698 """
699 if not os.path.isfile(_strFilename):
700 print "I cannot find the file %s" % _strFilename
701 raise
702 lInFile = open(_strFilename, "r").readlines()
703 self["_audit_creation_method"] = 'From 2-D detector using FIT2D and CIFfile'
704 self["_pd_meas_scan_method"] = "fixed"
705 self["_pd_spec_description"] = lInFile[0].strip()
706 try:
707 iLenData = int(lInFile[3])
708 except ValueError:
709 iLenData = None
710 lOneLoop = []
711 try:
712 f2ThetaMin = float(lInFile[4].split()[0])
713 last = ""
714 for sLine in lInFile[-20:]:
715 if sLine.strip() != "":
716 last = sLine.strip()
717 f2ThetaMax = float(last.split()[0])
718 limitsOK = True
719
720 except (ValueError, IndexError):
721 limitsOK = False
722 f2ThetaMin = 180.0
723 f2ThetaMax = 0
724
725 for sLine in lInFile[4:]:
726 sCleaned = sLine.split("#")[0].strip()
727 data = sCleaned.split()
728 if len(data) == 2 :
729 if not limitsOK:
730 f2Theta = float(data[0])
731 if f2Theta < f2ThetaMin :
732 f2ThetaMin = f2Theta
733 if f2Theta > f2ThetaMax :
734 f2ThetaMax = f2Theta
735 lOneLoop.append({ "_pd_meas_intensity_total": data[1] })
736 if not iLenData:
737 iLenData = len(lOneLoop)
738 assert (iLenData == len(lOneLoop))
739 self[ "_pd_meas_2theta_range_inc" ] = "%.4f" % ((f2ThetaMax - f2ThetaMin) / (iLenData - 1))
740 if self[ "_pd_meas_2theta_range_inc" ] < 0:
741 self[ "_pd_meas_2theta_range_inc" ] = abs (self[ "_pd_meas_2theta_range_inc" ])
742 tmp = f2ThetaMax
743 f2ThetaMax = f2ThetaMin
744 f2ThetaMin = tmp
745 self[ "_pd_meas_2theta_range_max" ] = "%.4f" % f2ThetaMax
746 self[ "_pd_meas_2theta_range_min" ] = "%.4f" % f2ThetaMin
747 self[ "_pd_meas_number_of_points" ] = str(iLenData)
748 self["loop_"] = [ [ ["_pd_meas_intensity_total" ], lOneLoop ] ]
749
750
751 @staticmethod
753 "Returns True if the key (string) exist in the array called loop"""
754 try:
755 loop.index(key)
756 return True
757 except ValueError:
758 return False
759