Line data Source code
1 : #include <alma/ASDMBinaries/SDMDataObjectReader.h>
2 :
3 : #include <iostream>
4 : #include <iomanip>
5 : #include <unistd.h>
6 : #include <sys/types.h>
7 : #include <sys/mman.h>
8 : #include <fcntl.h>
9 : #include <sys/stat.h>
10 : #include <errno.h>
11 : #include <exception>
12 : #include <sstream>
13 :
14 : #include <alma/ASDMBinaries/CommonDefines.h>
15 :
16 : #if defined( __APPLE__ )
17 : #include <AvailabilityMacros.h>
18 : // 10.5 defines stat64 but Tiger does NOT
19 : #if defined(MAC_OS_X_VERSION_10_4) && ! defined(MAC_OS_X_VERSION_10_5)
20 : #define stat64 stat
21 : #define fstat64 fstat
22 : #endif
23 : #endif
24 :
25 : using namespace CorrelationModeMod;
26 : using namespace CorrelatorTypeMod;
27 : using namespace PrimitiveDataTypeMod;
28 : using namespace ProcessorTypeMod;
29 :
30 : using namespace std;
31 :
32 : namespace asdmbinaries {
33 :
34 : const string SDMDataObjectReader::MIMEBOUNDARY_1 = $MIMEBOUNDARY1;
35 : const string SDMDataObjectReader::MIMEBOUNDARY_2 = $MIMEBOUNDARY2;
36 :
37 : #ifndef WITHOUT_BOOST
38 : const boost::regex SDMDataObjectReader::CONTENTIDBINREGEXP(" <([a-zA-Z]+)_([_0-9]+)\\.bin>");
39 : const boost::regex SDMDataObjectReader::CONTENTIDBINREGEXP1(" <([a-zA-Z]+)_([0-9]+)\\.bin>");
40 : const boost::regex SDMDataObjectReader::CONTENTIDBINREGEXP2(" <([a-zA-Z]+)_([0-9]+)_([0-9]+)\\.bin>");
41 : // const boost::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)\\.xml>");
42 : const boost::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)(_[0-9]+)?\\.xml>");
43 : const boost::regex SDMDataObjectReader::CONTENTIDDATASTRUCTUREREGEXP(" <DataStructure.xml>");
44 : #else
45 : const std::regex SDMDataObjectReader::CONTENTIDBINREGEXP(" <([a-zA-Z]+)_([_0-9]+)\\.bin>");
46 : const std::regex SDMDataObjectReader::CONTENTIDBINREGEXP1(" <([a-zA-Z]+)_([0-9]+)\\.bin>");
47 : const std::regex SDMDataObjectReader::CONTENTIDBINREGEXP2(" <([a-zA-Z]+)_([0-9]+)_([0-9]+)\\.bin>");
48 : // const std::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)\\.xml>");
49 : const std::regex SDMDataObjectReader::CONTENTIDSUBSETREGEXP(" <(Subset)_([0-9]+)(_[0-9]+)?\\.xml>");
50 : const std::regex SDMDataObjectReader::CONTENTIDDATASTRUCTUREREGEXP(" <DataStructure.xml>");
51 : #endif
52 : set<string> SDMDataObjectReader::BINATTACHNAMES;
53 : map<string, SDMDataObjectReader::BINATTACHCODES> SDMDataObjectReader::name2code;
54 : const bool SDMDataObjectReader::initClass_ = SDMDataObjectReader::initClass();
55 :
56 : // SDMDataObjectReader:: methods
57 : //
58 79 : SDMDataObjectReader::SDMDataObjectReader() {
59 79 : init();
60 79 : }
61 :
62 79 : SDMDataObjectReader::~SDMDataObjectReader() {
63 79 : done();
64 79 : }
65 :
66 1 : bool SDMDataObjectReader::initClass() {
67 1 : char* binAttachNamesC[] = {(char *) "actualDurations", (char *) "actualTimes", (char *) "autoData", (char *) "flags", (char *) "crossData", (char *) "zeroLags", 0};
68 7 : for (int i = 0; binAttachNamesC[i]; i++) {
69 6 : BINATTACHNAMES.insert(string(binAttachNamesC[i]));
70 : }
71 :
72 1 : name2code["actualDurations"] = ACTUALDURATIONS;
73 1 : name2code["actualTimes"] = ACTUALTIMES;
74 1 : name2code["autoData"] = AUTODATA;
75 1 : name2code["flags"] = FLAGS;
76 1 : name2code["crossData"] = CROSSDATA;
77 1 : name2code["zeroLags"] = ZEROLAGS;
78 :
79 1 : return true;
80 : }
81 :
82 79 : void SDMDataObjectReader::init() {
83 79 : filedes_ = 0;
84 79 : data_ = (char *) 0;
85 79 : position_ = 0;
86 79 : lastPosition_ = 0;
87 79 : endPosition_ = 0;
88 :
89 79 : read_ = UNKNOWN_;
90 :
91 79 : integrationNum_ = 0;
92 79 : }
93 :
94 1254303 : string::size_type SDMDataObjectReader::find(const string& s) {
95 :
96 : while (true) {
97 1254303 : while ((position_ < endPosition_) && (data_[position_++] != s.at(0))) {;}
98 :
99 20564 : if (position_ == endPosition_) return string::npos;
100 :
101 20564 : string::size_type lastPosition_ = position_;
102 20564 : unsigned long long int i = 1;
103 :
104 65100 : while ((position_ < endPosition_) && (i < s.length())) {
105 61128 : if (s.at(i) != data_[position_]) break;
106 44536 : i++;
107 44536 : position_++;
108 : }
109 :
110 20564 : if (i == s.length()) return (position_ - s.length());
111 :
112 16592 : if (position_ == endPosition_) return string::npos;
113 :
114 16592 : position_ = lastPosition_; //+1;
115 16592 : }
116 : }
117 :
118 1437 : bool SDMDataObjectReader::compare(const string& s) {
119 1437 : string::size_type index = position_;
120 :
121 1437 : unsigned int i = 0;
122 9489 : while ((index < endPosition_) && (i < s.length()) && (s.at(i++) == data_[index++])) {
123 : ;
124 : }
125 1437 : return (i == s.length());
126 : }
127 :
128 1437 : bool SDMDataObjectReader::EOD() {
129 1437 : return position_ >= endPosition_;
130 : }
131 :
132 :
133 732 : string SDMDataObjectReader::extractXMLHeader(const string& boundary) {
134 : // cout << "Entering extractXMLHeader" << endl;
135 :
136 1464 : string boundary_ = "\n--"+boundary;
137 :
138 732 : unsigned long int positionBeg = 0;
139 732 : unsigned long int positionEnd = 0;
140 :
141 : // We assume that we are at the first character of the body.
142 732 : positionBeg = position_;
143 :
144 : // Look for the next boundary supposed to be found right after an the XML header.
145 732 : if ( (positionEnd = find(boundary_)) == string::npos) {
146 0 : ostringstream eos;
147 0 : eos << "A second MIME boundary '" << boundary <<"' could not be found in the MIME message after position " << positionBeg << ".";
148 0 : throw SDMDataObjectReaderException(eos.str());
149 : }
150 :
151 732 : string xmlPart = string(data_+positionBeg, positionEnd-positionBeg);
152 :
153 : // cout << "Found XML Header : " << xmlPart << endl;
154 : // cout << "Exiting extractXMLHeader" << endl;
155 :
156 1464 : return xmlPart;
157 : }
158 :
159 :
160 3240 : void SDMDataObjectReader::tokenize(const string& str,
161 : vector<string>& tokens,
162 : const string& delimiters) {
163 : // Skip delimiters at beginning.
164 3240 : string::size_type lastPos = str.find_first_not_of(delimiters, 0);
165 : // Find first "non-delimiter".
166 3240 : string::size_type pos = str.find_first_of(delimiters, lastPos);
167 :
168 9720 : while (string::npos != pos || string::npos != lastPos) {
169 : // Found a token, add it to the vector.
170 6480 : tokens.push_back(str.substr(lastPos, pos - lastPos));
171 : // Skip delimiters. Note the "not_of"
172 6480 : lastPos = str.find_first_not_of(delimiters, pos);
173 : // Find next "non-delimiter"
174 6480 : pos = str.find_first_of(delimiters, lastPos);
175 : }
176 3240 : }
177 :
178 :
179 1437 : void SDMDataObjectReader::getFields(const string& header, map<string, string>& fields) {
180 : // We are at the last character of the line preceding the 1st field declaration.
181 2874 : istringstream iss;
182 1437 : iss.str(header);
183 2874 : string line;
184 :
185 1437 : getline(iss, line); // skip an nl
186 :
187 : // Split each line into field-name, field-value
188 4677 : while (getline(iss, line)) {
189 6480 : vector<string> nv;
190 3240 : tokenize(line, nv, ":");
191 3240 : if (nv.size() < 2) {
192 0 : ostringstream oss;
193 0 : oss << "Invalid field in a MIME header '" << header << "', integration #" << integrationNum_;
194 0 : throw SDMDataObjectReaderException (oss.str());
195 : }
196 3240 : fields[nv.at(0)] = nv.at(1);
197 : }
198 1437 : }
199 :
200 0 : string SDMDataObjectReader::getContentID() {
201 : // We are at the beginning of an attachment right after its opening boundary.
202 0 : string::size_type lastPosition = position_;
203 :
204 : // Now let's extract the header (which is followed by two successive nl nl)
205 : string::size_type endHeader;
206 0 : if ( (endHeader = find ("\n\n")) == string::npos) {
207 0 : throw SDMDataObjectReaderException("Could not find the end of a MIME header");
208 : }
209 :
210 0 : string header(data_+lastPosition, (endHeader - lastPosition));
211 :
212 0 : map<string, string> fields;
213 0 : getFields(header, fields);
214 :
215 : // Look for the Content-ID field
216 0 : map<string, string>::iterator iter = fields.find("Content-ID");
217 0 : if (iter == fields.end()) {
218 0 : ostringstream oss;
219 0 : oss << "'Content-ID' field is missing the MIME header of an attachment (approx. char position = " << position_ << ").";
220 0 : throw SDMDataObjectReaderException(oss.str());
221 : }
222 0 : return iter->second;
223 : }
224 :
225 1437 : string SDMDataObjectReader::getContentLocation() {
226 : // We are at the beginning of an attachment right after its opening boundary.
227 1437 : string::size_type lastPosition = position_;
228 :
229 : // Now let's extract the header (which is followed by two successive nl nl)
230 : string::size_type endHeader;
231 1437 : if ( (endHeader = find ("\n\n")) == string::npos) {
232 0 : throw SDMDataObjectReaderException("Could not find the end of a MIME header");
233 : }
234 :
235 2874 : string header(data_+lastPosition, (endHeader - lastPosition));
236 :
237 2874 : map<string, string> fields;
238 1437 : getFields(header, fields);
239 :
240 : // Look for the Content-Location field
241 1437 : map<string, string>::iterator iter = fields.find("Content-Location");
242 1437 : if (iter == fields.end()) {
243 0 : ostringstream oss;
244 0 : oss << "'Content-Location' field is missing the MIME header of an attachment (approx. char position = " << position_ << ").";
245 0 : throw SDMDataObjectReaderException(oss.str());
246 : }
247 2874 : return iter->second;
248 : }
249 :
250 705 : void SDMDataObjectReader::processBinaryAttachment(const string& boundary, const SDMDataSubset& sdmDataSubset) {
251 :
252 1410 : string contentLocation = getContentLocation();
253 :
254 : // Extract the attachment name and the integration number.
255 : #ifndef WITHOUT_BOOST
256 : boost::cmatch what;
257 : boost::regex r(" *" + sdmDataSubset.projectPath()+"(actualDurations|actualTimes|autoData|flags|crossData|zeroLags)\\.bin");
258 : if (boost::regex_match(contentLocation.c_str(), what, r)) {
259 : }
260 : #else
261 1410 : std::cmatch what;
262 2115 : std::regex r(" *" + sdmDataSubset.projectPath()+"(actualDurations|actualTimes|autoData|flags|crossData|zeroLags)\\.bin");
263 705 : if (std::regex_match(contentLocation.c_str(), what, r)) {
264 : }
265 : #endif
266 : else {
267 0 : ostringstream oss;
268 0 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of an attachment (approx. char position = " << position_ << ").";
269 0 : throw SDMDataObjectReaderException(oss.str());
270 : }
271 :
272 : // We are at the first byte of the body of the binary attachment.
273 705 : string::size_type startAttachPosition = position_; // mcaillat
274 :
275 : // Now look for its closing boundary (or opening boundary of the next one if any).
276 1410 : string boundary_ = "\n--"+boundary;
277 : string::size_type endAttachPosition;
278 705 : if ( (endAttachPosition = find(boundary_)) == string::npos) {
279 0 : ostringstream eos;
280 0 : eos << "A MIME boundary '" << boundary << "' terminating a binary attachment starting approx. at character " << startAttachPosition <<" could not be found.";
281 0 : throw SDMDataObjectReaderException(eos.str());
282 : }
283 :
284 : // Compute the length of the attachment.
285 705 : string::size_type length = endAttachPosition - startAttachPosition;
286 :
287 :
288 :
289 705 : switch (name2code[what[1]]) {
290 0 : case ACTUALDURATIONS:
291 0 : actualDurations_ = (ACTUALDURATIONSTYPE *) (data_ + startAttachPosition);
292 0 : nActualDurations_ = length / sizeof (ACTUALDURATIONSTYPE);
293 0 : attachmentFlags.set(ACTUALDURATIONS);
294 : //cout << "actualDurations =" << (unsigned long int) actualDurations_ << ", nActualDurations = " << nActualDurations_ << endl;
295 0 : break;
296 0 : case ACTUALTIMES:
297 0 : actualTimes_ = (ACTUALTIMESTYPE *) (data_ + startAttachPosition);
298 0 : nActualTimes_ = length / sizeof( ACTUALTIMESTYPE);
299 0 : attachmentFlags.set(ACTUALTIMES);
300 : //cout << "actualTimes =" << (unsigned long int) actualTimes_ << ", nActualTimes = " << nActualTimes_ << endl;
301 0 : break;
302 366 : case AUTODATA:
303 366 : autoData_ = (AUTODATATYPE *) (data_ + startAttachPosition);
304 366 : nAutoData_ = length / sizeof(AUTODATATYPE);
305 366 : attachmentFlags.set(AUTODATA);
306 : //cout << "autoData =" << (unsigned long int) autoData_ << ", nAutoData = " << nAutoData_ << endl;
307 366 : break;
308 339 : case FLAGS:
309 339 : flags_ = (const FLAGSTYPE *) (data_ + startAttachPosition);
310 339 : nFlags_ = length / sizeof(FLAGSTYPE);
311 339 : attachmentFlags.set(FLAGS);
312 : //cout << "flags =" << (unsigned long int) flags_ << ", nFlags = " << nFlags_ << endl;
313 339 : break;
314 0 : case CROSSDATA:
315 0 : shortCrossData_ = 0;
316 0 : longCrossData_ = 0;
317 0 : floatCrossData_ = 0;
318 0 : switch (sdmDataSubset.crossDataType()) {
319 0 : case INT16_TYPE:
320 0 : shortCrossData_ = (const SHORTCROSSDATATYPE*) (data_ + startAttachPosition);
321 0 : nCrossData_ = length / sizeof (SHORTCROSSDATATYPE);
322 : //cout << "shortCrossData = " << (unsigned long int) shortCrossData_ << ", nShortCrossData = " << nCrossData_ << endl;
323 0 : break;
324 :
325 0 : case INT32_TYPE:
326 0 : longCrossData_ = (const INTCROSSDATATYPE*) (data_ + startAttachPosition);
327 0 : nCrossData_ = length / sizeof (INTCROSSDATATYPE);
328 : //cout << "longCrossData = " << (unsigned long int) longCrossData_ << ", nLongCrossData = " << nCrossData_ << endl;
329 0 : break;
330 :
331 0 : case FLOAT32_TYPE:
332 0 : floatCrossData_ = (const FLOATCROSSDATATYPE*) (data_ + startAttachPosition);
333 0 : nCrossData_ = length / sizeof (FLOATCROSSDATATYPE);
334 0 : break;
335 :
336 0 : default:
337 0 : ostringstream eos;
338 0 : eos << "'" << CPrimitiveDataType::toString(sdmDataSubset.crossDataType())
339 0 : << "' is not a valid primitive data type for CROSSDATA";
340 0 : throw SDMDataObjectReaderException(eos.str());
341 : // cout << "Unrecognized type for cross data" << endl;
342 : }
343 0 : attachmentFlags.set(CROSSDATA);
344 0 : break;
345 0 : case ZEROLAGS:
346 0 : zeroLags_ = (ZEROLAGSTYPE *) (data_ + startAttachPosition);
347 0 : nZeroLags_ = length / sizeof(ZEROLAGSTYPE) ;
348 0 : attachmentFlags.set(ZEROLAGS);
349 : //cout << "zeroLags =" << (unsigned long int) zeroLags_ << ", nZeroLags = " << nZeroLags_ << endl;
350 0 : break;
351 :
352 0 : default:
353 : //cout << "Unrecognized code" << endl;
354 0 : break;
355 : }
356 : // cout << "found a binary attachment" << endl;
357 705 : }
358 :
359 366 : void SDMDataObjectReader::processMIMESDMDataHeader() {
360 : // cout << "Entering processMIMESDMDataHeader" << endl;
361 732 : string sdmDataHeader = extractXMLHeader(MIMEBOUNDARY_1);
362 366 : parser_.parseMemoryHeader(sdmDataHeader, sdmDataObject_);
363 : // cout << "Exiting processMIMESDMDataHeader" << endl;
364 366 : }
365 :
366 366 : void SDMDataObjectReader::processMIMESDMDataSubsetHeader(SDMDataSubset& sdmDataSubset) {
367 : // cout << "Entering processMIMESDMDataSubsetHeader" << endl;
368 732 : string sdmDataSubsetHeader = extractXMLHeader(MIMEBOUNDARY_2);
369 366 : if (sdmDataSubset.owner_->isCorrelation())
370 0 : parser_.parseMemoryCorrSubsetHeader(sdmDataSubsetHeader, sdmDataSubset);
371 : else
372 366 : parser_.parseMemoryTPSubsetHeader(sdmDataSubsetHeader, sdmDataSubset);
373 : // cout << "Exiting processMIMESDMDataSubsetHeader" << endl;
374 366 : }
375 :
376 0 : void SDMDataObjectReader::processMIMEIntegration() {
377 : // cout << "Entering processMIMEIntegration" << endl;
378 :
379 : // We are 1 character beyond the end of --<MIMEBOUNDARY_2>
380 0 : string contentLocation = getContentLocation();
381 :
382 : #ifndef WITHOUT_BOOST
383 : boost::regex r(" *" + sdmDataObject_.projectPath() + "([[:digit:]]+/){1,2}" + "desc.xml");;
384 : ostringstream oss;
385 :
386 : // Extract the Subset name and the integration [, subintegration] number.
387 : boost::cmatch what;
388 : if (!boost::regex_match(contentLocation.c_str(), what, r)) {
389 : ostringstream oss;
390 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of a Subset (approx. char position = " << position_ << ").";
391 : throw SDMDataObjectReaderException(oss.str());
392 : }
393 : #else
394 0 : std::regex r(" *" + sdmDataObject_.projectPath() + "([[:digit:]]+/){1,2}" + "desc.xml");;
395 0 : ostringstream oss;
396 :
397 : // Extract the Subset name and the integration [, subintegration] number.
398 0 : std::cmatch what;
399 0 : if (!std::regex_match(contentLocation.c_str(), what, r)) {
400 0 : ostringstream oss;
401 0 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of a Subset (approx. char position = " << position_ << ").";
402 0 : throw SDMDataObjectReaderException(oss.str());
403 : }
404 : #endif
405 :
406 0 : SDMDataSubset integration(&sdmDataObject_);
407 :
408 : // The SDMDataSubset header.
409 0 : processMIMESDMDataSubsetHeader(integration);
410 :
411 0 : if (integration.aborted_) {
412 : // The [sub]integration has been aborted. Just append its header, without trying to get binary attachment.
413 0 : sdmDataObject_.append(integration);
414 : }
415 : else {
416 : // This is regular [sub]integration, process its binary attachments.
417 0 : attachmentFlags.reset();
418 0 : while (!EOD() && !compare("--")) {
419 0 : processBinaryAttachment(MIMEBOUNDARY_2, integration);
420 : }
421 :
422 0 : if (EOD())
423 0 : throw SDMDataObjectReaderException("Unexpected end of data");
424 :
425 : // Now check if the binary attachments found are compatible with the correlation mode
426 : // and if their sizes are equal to what is announced in the global header.
427 :
428 0 : if (!attachmentFlags.test(ACTUALDURATIONS)) {
429 : // No actualdurations attachment found, ok then set everything to 0.
430 0 : integration.actualDurations_ = 0;
431 0 : integration.nActualDurations_ = 0;
432 : }
433 : else {
434 0 : if (nActualDurations_ != sdmDataObject_.dataStruct().actualDurations().size()) {
435 0 : ostringstream oss;
436 0 : oss << "Data subset '"<<contentLocation<<"': ";
437 0 : oss << "zize of 'actualDuration' attachment (" << nActualDurations_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualDurations().size() << ")";
438 0 : throw SDMDataObjectReaderException(oss.str());
439 : }
440 :
441 0 : integration.actualDurations_ = actualDurations_;
442 0 : integration.nActualDurations_ = nActualDurations_;
443 : }
444 :
445 0 : if (!attachmentFlags.test(ACTUALTIMES)) {
446 : // No actualtimes attachment found, ok then set everything to 0.
447 0 : integration.actualTimes_ = 0;
448 0 : integration.nActualTimes_ = 0;
449 : }
450 : else {
451 0 : if (nActualTimes_ != sdmDataObject_.dataStruct().actualTimes().size()) {
452 0 : ostringstream oss;
453 0 : oss << "Data subset '"<<contentLocation<<"': ";
454 0 : oss << "size of 'actualTimes' attachment (" << nActualTimes_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualTimes().size() << ")";
455 0 : throw SDMDataObjectReaderException(oss.str());
456 : }
457 0 : integration.actualTimes_ = actualTimes_;
458 0 : integration.nActualTimes_ = nActualTimes_;
459 : }
460 :
461 : // The flags are optional. They may be absent.
462 0 : if (!attachmentFlags.test(FLAGS)) {
463 : // No flags binary attachment found, ok then set everything to 0.
464 0 : integration.flags_ = 0;
465 0 : integration.nFlags_ = 0;
466 : }
467 : else {
468 : // flags found
469 : // Check size conformity
470 0 : if (nFlags_ != sdmDataObject_.dataStruct().flags().size()) {
471 0 : ostringstream oss;
472 0 : oss << "Data subset '"<<contentLocation<<"': ";
473 0 : oss << "size of 'flags' attachment (" << nFlags_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().flags().size() << ")";
474 0 : throw SDMDataObjectReaderException(oss.str());
475 : }
476 :
477 : // Set pointer and size
478 0 : integration.flags_ = flags_;
479 0 : integration.nFlags_ = nFlags_;
480 : }
481 :
482 :
483 : //
484 : // The presence of crossData and autoData depends on the correlation mode.
485 :
486 : // crossDataTypeSize is not currently used
487 : // unsigned int crossDataTypeSize;
488 0 : switch (sdmDataObject_.correlationMode()) {
489 0 : case CROSS_ONLY:
490 0 : if (!attachmentFlags.test(CROSSDATA)) {
491 0 : ostringstream oss;
492 0 : oss << "Data subset '"<<contentLocation<<"': ";
493 0 : oss << "a binary attachment 'crossData' was expected in integration #" << integrationNum_;
494 0 : throw SDMDataObjectReaderException(oss.str());
495 : }
496 :
497 :
498 0 : if (nCrossData_ != sdmDataObject_.dataStruct().crossData().size()) {
499 0 : ostringstream oss;
500 0 : oss << "Data subset '"<<contentLocation<<"': ";
501 0 : oss << "size of 'crossData' attachment (" << nCrossData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().crossData().size() << ")";
502 0 : throw SDMDataObjectReaderException(oss.str());
503 : }
504 :
505 :
506 0 : if (attachmentFlags.test(AUTODATA)) {
507 0 : ostringstream oss;
508 0 : oss << "Data subset '"<<contentLocation<<"': ";
509 0 : oss << "found an unexpected attachment 'autoData' in integration #" << integrationNum_ << ".";
510 0 : throw SDMDataObjectReaderException(oss.str());
511 : }
512 :
513 0 : integration.shortCrossData_ = 0;
514 0 : integration.longCrossData_ = 0;
515 0 : integration.floatCrossData_ = 0;
516 0 : integration.nCrossData_ = 0;
517 0 : switch (integration.crossDataType()) {
518 0 : case INT32_TYPE:
519 0 : integration.longCrossData_ = longCrossData_;
520 0 : integration.nCrossData_ = nCrossData_;
521 0 : break;
522 :
523 0 : case INT16_TYPE:
524 0 : integration.shortCrossData_ = shortCrossData_;
525 0 : integration.nCrossData_ = nCrossData_;
526 0 : break;
527 :
528 0 : case FLOAT32_TYPE:
529 0 : integration.floatCrossData_ = floatCrossData_;
530 0 : integration.nCrossData_ = nCrossData_;
531 0 : break;
532 :
533 0 : default:
534 0 : throw SDMDataObjectReaderException("'"+CPrimitiveDataType::name(integration.crossDataType())+"' unexpected here.");
535 : }
536 0 : break;
537 :
538 0 : case AUTO_ONLY:
539 0 : if (!attachmentFlags.test(AUTODATA)) {
540 0 : ostringstream oss;
541 0 : oss << "Data subset '"<<contentLocation<<"': ";
542 0 : oss << "a binary attachment 'autoData' was expected.";
543 0 : throw SDMDataObjectReaderException(oss.str());
544 : }
545 0 : if (nAutoData_ != sdmDataObject_.dataStruct().autoData().size()) {
546 0 : ostringstream oss;
547 0 : oss << "Data subset '"<<contentLocation<<"': ";
548 0 : oss << "size of 'autoData' attachment (" << nAutoData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().autoData().size() << ")";
549 0 : throw SDMDataObjectReaderException(oss.str());
550 : }
551 :
552 :
553 0 : if (attachmentFlags.test(CROSSDATA)) {
554 0 : ostringstream oss;
555 0 : oss << "Data subset '"<<contentLocation<<"': ";
556 0 : oss << "found an unexpected attachment 'crossData'";
557 0 : throw SDMDataObjectReaderException(oss.str());
558 : }
559 :
560 0 : integration.autoData_ = autoData_;
561 0 : integration.nAutoData_ = nAutoData_;
562 :
563 0 : break;
564 :
565 0 : case CROSS_AND_AUTO:
566 0 : if (!attachmentFlags.test(AUTODATA)) {
567 0 : ostringstream oss;
568 0 : oss << "Data subset '"<<contentLocation<<"': ";
569 0 : oss << "a binary attachment 'autoData' was expected.";
570 0 : throw SDMDataObjectReaderException(oss.str());
571 : }
572 0 : if (nAutoData_ != sdmDataObject_.dataStruct().autoData().size()) {
573 0 : ostringstream oss;
574 0 : oss << "Data subset '"<<contentLocation<<"': ";
575 0 : oss << "size of 'autoData' attachment (" << nAutoData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().autoData().size() << ")";
576 0 : throw SDMDataObjectReaderException(oss.str());
577 : }
578 :
579 0 : if (!attachmentFlags.test(CROSSDATA)) {
580 0 : ostringstream oss;
581 0 : oss << "Data subset '"<<contentLocation<<"': ";
582 0 : oss << "a binary attachment 'crossData' was expected.";
583 0 : throw SDMDataObjectReaderException(oss.str());
584 : }
585 :
586 : // crossDataTypeSize is not currently used, skip this block
587 : // switch (integration.crossDataType()) {
588 : // case INT32_TYPE:
589 : // crossDataTypeSize = sizeof(INTCROSSDATATYPE);
590 : // break;
591 : // case INT16_TYPE:
592 : // crossDataTypeSize = sizeof(SHORTCROSSDATATYPE);
593 : // break;
594 : // case FLOAT32_TYPE:
595 : // crossDataTypeSize = sizeof(FLOATCROSSDATATYPE);
596 : // break;
597 : // default:
598 : // throw SDMDataObjectReaderException("'"+CPrimitiveDataType::name(integration.crossDataType())+"' unexpected here.");
599 : // }
600 :
601 0 : if (nCrossData_ != sdmDataObject_.dataStruct().crossData().size()) {
602 0 : ostringstream oss;
603 0 : oss << "Data subset '"<<contentLocation<<"': ";
604 0 : oss << "size of 'crossData' attachment (" << nCrossData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().crossData().size() << ")";
605 0 : throw SDMDataObjectReaderException(oss.str());
606 : }
607 :
608 :
609 0 : integration.shortCrossData_ = 0;
610 0 : integration.longCrossData_ = 0;
611 0 : integration.floatCrossData_ = 0;
612 0 : integration.nCrossData_ = 0;
613 0 : switch (integration.crossDataType()) {
614 0 : case INT32_TYPE:
615 0 : integration.longCrossData_ = longCrossData_;
616 0 : integration.nCrossData_ = nCrossData_;
617 0 : break;
618 :
619 0 : case INT16_TYPE:
620 0 : integration.shortCrossData_ = shortCrossData_;
621 0 : integration.nCrossData_ = nCrossData_;
622 0 : break;
623 :
624 0 : case FLOAT32_TYPE:
625 0 : integration.floatCrossData_ = floatCrossData_;
626 0 : integration.nCrossData_ = nCrossData_;
627 0 : break;
628 :
629 0 : default:
630 0 : throw SDMDataObjectReaderException("Data subset '"+contentLocation+"': '"+CPrimitiveDataType::name(integration.crossDataType())+"' unexpected here.");
631 : }
632 :
633 0 : integration.autoData_ = autoData_;
634 0 : integration.nAutoData_ = nAutoData_;
635 :
636 :
637 0 : break;
638 0 : default:
639 0 : throw SDMDataObjectReaderException("Data subset '"+contentLocation+"': unrecognized correlation mode");
640 : break;
641 : }
642 :
643 :
644 0 : if (attachmentFlags.test(ZEROLAGS)) {
645 : // Refuse the zeroLags attachment if it's not a Correlator or if the correlator is a CORRELATOR_FX (ACA).
646 0 : if ((sdmDataObject_.processorType_ != CORRELATOR) || (sdmDataObject_.correlatorType() == FX))
647 0 : throw SDMDataObjectReaderException("zeroLags are not expected from a correlator CORRELATOR_FX");
648 :
649 0 : if (nZeroLags_ != sdmDataObject_.dataStruct().zeroLags().size()) {
650 0 : ostringstream oss;
651 0 : oss << "Data subset '"<<contentLocation<<"': ";
652 0 : oss << "size of 'zeroLags' attachment (" << nZeroLags_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().zeroLags().size() << ")";
653 0 : throw SDMDataObjectReaderException(oss.str());
654 : }
655 0 : integration.zeroLags_ = zeroLags_;
656 0 : integration.nZeroLags_ = nZeroLags_;
657 : }
658 :
659 0 : sdmDataObject_.append(integration);
660 : }
661 :
662 0 : if (!compare("--\n--"+MIMEBOUNDARY_1))
663 0 : throw SDMDataObjectReaderException("Data subset '"+contentLocation+"': expecting a '--"+MIMEBOUNDARY_1+"' at the end of a data subset");
664 :
665 0 : find("--\n--"+MIMEBOUNDARY_1);
666 : // cout << "Exiting processMIMEIntegration" << endl;
667 0 : }
668 :
669 0 : void SDMDataObjectReader::processMIMEIntegrations() {
670 : // cout << "Entering processMIMEIntegrations" << endl;
671 :
672 0 : while (!compare("--")) {
673 : // We are one character beyond the last character of --<MIMEBOUNDARY_1>
674 : // Let's ignore the MIME Header right after the --<MIMEBOUNDARY_1>
675 : // and move to the next --<MIMEBOUNDARY_2>
676 0 : if (find("--"+MIMEBOUNDARY_2) == string::npos)
677 0 : throw SDMDataObjectReaderException("Expecting a boundary '--"+MIMEBOUNDARY_2+"' at this position");
678 :
679 : //Process the next integration.
680 0 : integrationNum_++;
681 0 : processMIMEIntegration() ;
682 : }
683 :
684 : // cout << "Exiting processMIMEIntegrations" << endl;
685 0 : }
686 :
687 366 : void SDMDataObjectReader::processMIMESubscan() {
688 : // cout << "Entering processMIMESubscan" << endl;
689 :
690 : // We are one character beyond the last character of --<MIMEBOUNDARY_1>
691 : // Let's ignore the MIME Header right after the --<MIMEBOUNDARY_1>
692 : // and move to the next --<MIMEBOUNDARY_2>
693 : // We are 1 character beyond the end of --<MIMEBOUNDARY_2>
694 366 : if (find("--"+MIMEBOUNDARY_2) == string::npos)
695 0 : throw SDMDataObjectReaderException("Expecting a boundary '--"+MIMEBOUNDARY_2+"' at this position");
696 :
697 :
698 732 : string contentLocation = getContentLocation();
699 : // cout << contentLocation << endl;
700 : #ifndef WITHOUT_BOOST
701 : boost::regex r(" *"+sdmDataObject_.projectPath() + "desc.xml");
702 :
703 : // Extract the Subset name and the suffix number (which must be equal to 1).
704 : boost::cmatch what;
705 : if (!boost::regex_match(contentLocation.c_str(), what, r)) {
706 : ostringstream oss;
707 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of the subset";
708 : throw SDMDataObjectReaderException(oss.str());
709 : }
710 : #else
711 1098 : std::regex r(" *"+sdmDataObject_.projectPath() + "desc.xml");
712 :
713 : // Extract the Subset name and the suffix number (which must be equal to 1).
714 732 : std::cmatch what;
715 366 : if (!std::regex_match(contentLocation.c_str(), what, r)) {
716 0 : ostringstream oss;
717 0 : oss << "Invalid Content-Location field '" << contentLocation <<"' in MIME header of the subset";
718 0 : throw SDMDataObjectReaderException(oss.str());
719 : }
720 : #endif
721 :
722 366 : SDMDataSubset subscan(&sdmDataObject_);
723 :
724 : // The SDMDataSubset header.
725 366 : processMIMESDMDataSubsetHeader(subscan);
726 :
727 :
728 : // Process the binary attachments.
729 366 : attachmentFlags.reset();
730 366 : integrationNum_++;
731 1071 : while(!EOD() && !compare("--")) {
732 705 : processBinaryAttachment(MIMEBOUNDARY_2, subscan);
733 : }
734 :
735 366 : if (EOD()) throw SDMDataObjectReaderException("Unexpected end of data");
736 :
737 : // Now check if the binary attachments found are compatible with the correlation mode.
738 : // and if their sizes are equal to what is announced in the global header.
739 :
740 : // Start with the only mandatory attachment : autoData.
741 366 : if (!attachmentFlags.test(AUTODATA)) {
742 0 : ostringstream oss;
743 0 : oss << "Binary attachment 'autoData' was expected in integration #" << integrationNum_;
744 0 : throw SDMDataObjectReaderException(oss.str());
745 : }
746 366 : if (nAutoData_ != sdmDataObject_.dataStruct().autoData().size()) {
747 0 : ostringstream oss;
748 0 : oss << "Size of 'autoData' attachment (" << nAutoData_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().autoData().size() << ")";
749 0 : throw SDMDataObjectReaderException(oss.str());
750 : }
751 :
752 366 : subscan.autoData_ = autoData_;
753 366 : subscan.nAutoData_ = nAutoData_;
754 :
755 :
756 : // And now consider the optional attachments.
757 :
758 : // The actualDurations are optional. They may be absent.
759 366 : if (!attachmentFlags.test(ACTUALDURATIONS)) {
760 : // No actualDurations binary attachment found, ok then set everything to 0.
761 366 : subscan.actualDurations_ = 0;
762 366 : subscan.nActualDurations_ = 0;
763 : }
764 : else {
765 : // actualDurations found
766 : // Check size conformity
767 0 : if (nActualDurations_ != sdmDataObject_.dataStruct().actualDurations().size()) {
768 0 : ostringstream oss;
769 0 : oss << "Size of 'actualDurations' attachment (" << nActualDurations_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualDurations().size() << ")";
770 0 : throw SDMDataObjectReaderException(oss.str());
771 : }
772 :
773 : // Set pointer and size
774 0 : subscan.actualDurations_ = actualDurations_;
775 0 : subscan.nActualDurations_ = nActualDurations_;
776 : }
777 :
778 :
779 : // The actualTimes are optional. They may be absent.
780 366 : if (!attachmentFlags.test(ACTUALTIMES)) {
781 : // No actualTimes binary attachment found, ok then set everything to 0.
782 366 : subscan.actualTimes_ = 0;
783 366 : subscan.nActualTimes_ = 0;
784 : }
785 : else {
786 : // actualTimes found
787 : // Check size conformity
788 0 : if (nActualTimes_ != sdmDataObject_.dataStruct().actualTimes().size()) {
789 0 : ostringstream oss;
790 0 : oss << "Size of 'actualTimes' attachment (" << nActualTimes_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().actualTimes().size() << ")";
791 0 : throw SDMDataObjectReaderException(oss.str());
792 : }
793 :
794 : // Set pointer and size
795 0 : subscan.actualTimes_ = actualTimes_;
796 0 : subscan.nActualTimes_ = nActualTimes_;
797 : }
798 :
799 :
800 : // The flags are optional. They may be absent.
801 366 : if (!attachmentFlags.test(FLAGS)) {
802 : // No flags binary attachment found, ok then set everything to 0.
803 27 : subscan.flags_ = 0;
804 27 : subscan.nFlags_ = 0;
805 : }
806 : else {
807 : // flags found
808 : // Check size conformity
809 339 : if (nFlags_ != sdmDataObject_.dataStruct().flags().size()) {
810 0 : ostringstream oss;
811 0 : oss << "Size of 'flags' attachment (" << nFlags_ << ") is not equal to the size announced in the header (" << sdmDataObject_.dataStruct().flags().size() << ")";
812 0 : throw SDMDataObjectReaderException(oss.str());
813 : }
814 :
815 : // Set pointer and size
816 339 : subscan.flags_ = flags_;
817 339 : subscan.nFlags_ = nFlags_;
818 : }
819 :
820 : // And finally let's check that no unexpected binary attachment was found.
821 366 : if (attachmentFlags.test(CROSSDATA)) {
822 0 : ostringstream oss;
823 0 : oss << "Found an unexpected attachment 'crossData' in the binary attachments.";
824 0 : throw SDMDataObjectReaderException(oss.str());
825 : }
826 :
827 :
828 366 : if (attachmentFlags.test(ZEROLAGS)) {
829 0 : ostringstream oss;
830 0 : oss << "Found an unexpected attachment 'zeroLags' in the binary attachments.";
831 0 : throw SDMDataObjectReaderException(oss.str());
832 : }
833 :
834 366 : sdmDataObject_.tpDataSubset(subscan);
835 :
836 :
837 366 : if (!compare("--\n--"+MIMEBOUNDARY_1))
838 0 : throw SDMDataObjectReaderException("Expecting a '--"+MIMEBOUNDARY_1+"' at the end of a data subset");
839 :
840 366 : find("--\n--"+MIMEBOUNDARY_1);
841 366 : }
842 :
843 366 : void SDMDataObjectReader::processMIME() {
844 : // cout << "Entering processMIME" << endl;
845 : // Let's find the opening boundary
846 366 : if (find("--"+MIMEBOUNDARY_1) == string::npos)
847 0 : throw SDMDataObjectReaderException("Expecting a first boundary '--"+MIMEBOUNDARY_1+"'.");
848 :
849 : // Check the Content-Location of the MIME header ...but do nothing special with it...
850 732 : string contentLocation=getContentLocation();
851 :
852 : // Detect SDMDataHeader.
853 366 : processMIMESDMDataHeader();
854 :
855 : // Do we have packed data (i.e. only one SDMDataSubset grouping all the integrations) or one integration per SDMDataSubset (i.e. per timestamp) ?
856 366 : if (sdmDataObject_.hasPackedData()) {
857 366 : processMIMESubscan();
858 : }
859 : else {
860 0 : processMIMEIntegrations();
861 : }
862 :
863 :
864 : // if (sdmDataObject_.isCorrelation()) {
865 : // // Process integrations.
866 : // processMIMEIntegrations();
867 : // }
868 : // else if (sdmDataObject_.isTP()){
869 : // // cout << "TP data" << endl;
870 : // if (sdmDataObject_.dimensionality() == 0) {
871 : // processMIMESubscan();
872 : // }
873 : // else {
874 : // processMIMEIntegrations();
875 : // }
876 : // }
877 : // else {
878 : // // cout << "Unrecognized type of binary data." << endl;
879 : // }
880 : //cout << "Exiting processMIME" << endl;
881 366 : }
882 :
883 366 : const SDMDataObject & SDMDataObjectReader::read(const char * buffer, unsigned long int size, bool fromFile) {
884 366 : if (!fromFile) read_ = MEMORY_;
885 :
886 : // Set up all sensitive pointers and sizes.
887 366 : data_ = (char *)buffer;
888 366 : dataSize_ = size;
889 366 : position_ = 0;
890 366 : endPosition_ = size;
891 :
892 : // And process the MIME message
893 366 : sdmDataObject_.valid_ = true;
894 366 : processMIME();
895 :
896 366 : sdmDataObject_.owns();
897 366 : return sdmDataObject_;
898 : }
899 :
900 0 : const SDMDataObject& SDMDataObjectReader::ref() const {
901 0 : if (read_ == UNKNOWN_)
902 0 : throw SDMDataObjectReaderException("a reference to an SDMDataObject cannot be obtained as long as the method 'read' has not been called.");
903 0 : return sdmDataObject_;
904 : }
905 :
906 0 : const SDMDataObject* SDMDataObjectReader::ptr() const {
907 0 : if (read_ == UNKNOWN_)
908 0 : throw SDMDataObjectReaderException("a reference to an SDMDataObject cannot be obtained as long as the method 'read' has not been called.");
909 0 : return &sdmDataObject_;
910 : }
911 :
912 445 : void SDMDataObjectReader::done() {
913 445 : switch (read_) {
914 79 : case UNKNOWN_:
915 : case MEMORY_:
916 79 : break;
917 366 : case FILE_:
918 366 : munmap((caddr_t) data_, dataSize_);
919 366 : close(filedes_);
920 366 : break;
921 : }
922 445 : sdmDataObject_.done();
923 445 : read_ = UNKNOWN_;
924 445 : }
925 :
926 0 : const SDMDataObject& SDMDataObjectReader::sdmDataObject() {
927 0 : return sdmDataObject_;
928 : }
929 :
930 :
931 366 : const SDMDataObject & SDMDataObjectReader::read(string filename) {
932 :
933 : struct stat fattr; // stat64 fattr;
934 :
935 : unsigned long int filesize;
936 366 : char* data = 0;
937 : // Open the file.
938 366 : errno = 0;
939 : #ifdef __APPLE__
940 : if ( (filedes_ = open(filename.c_str(), O_RDONLY )) == -1) {
941 : #else
942 366 : if ( (filedes_ = open(filename.c_str(), O_RDONLY | O_LARGEFILE)) == -1) {
943 : #endif
944 0 : string message(strerror(errno));
945 0 : throw SDMDataObjectReaderException("Could not open file '" + filename + "'. The message was '" + message + "'");
946 : }
947 366 : read_ = FILE_;
948 :
949 : // Get its size.
950 366 : errno = 0;
951 366 : int status = fstat(filedes_,&fattr); // fstat64(filedes_,&fattr);
952 366 : if (status == -1) {
953 0 : string message(strerror(errno));
954 0 : throw SDMDataObjectReaderException("Could not retrieve size of file '" + filename + "'. The message was '" + message + "'");
955 : }
956 366 : filesize = fattr.st_size;
957 :
958 : // Map it to virtual memory address space.
959 366 : errno = 0;
960 :
961 366 : data = (char *) mmap((caddr_t)0, filesize, PROT_READ, MAP_SHARED, filedes_, (off_t)0);
962 366 : if ( ((unsigned long) data) == 0xffffffff) {
963 0 : string message(strerror(errno));
964 0 : throw SDMDataObjectReaderException("Could not map file '" + filename + "' to memory. The message was '" + message + "'");
965 : }
966 :
967 : // cout << "Successfully mapped file." << endl;
968 : // And delegate to the other read (memory buffer) method.
969 366 : return read(data, filesize, true);
970 : }
971 :
972 : // SDMDataObjectReader::
973 :
974 : } // namespace asdmbinaries
975 :
|
