Line data Source code
1 : #include <alma/ASDMBinaries/SDMDataObjectStreamReader.h>
2 :
3 : #ifdef WITHOUT_BOOST
4 : #include <regex>
5 : #include <alma/ASDM/Misc.h>
6 : #endif
7 :
8 : using namespace CorrelationModeMod;
9 : using namespace CorrelatorTypeMod;
10 : using namespace PrimitiveDataTypeMod;
11 : using namespace ProcessorTypeMod;
12 :
13 : using namespace std;
14 :
15 : namespace asdmbinaries {
16 :
17 0 : SDMDataObjectStreamReaderException::SDMDataObjectStreamReaderException():message("SDMDataObjectStreamReaderException:") {;}
18 :
19 0 : SDMDataObjectStreamReaderException::SDMDataObjectStreamReaderException(const string & message): message("SDMDataObjectStreamReaderException:" + message) {;}
20 :
21 0 : const string& SDMDataObjectStreamReaderException::getMessage() { return message; }
22 :
23 1903 : SDMDataObjectStreamReader::SDMDataObjectStreamReader() {
24 : // cout << "SDMDataObjectStreamReader::SDMDataObjectStreamReader() : entering" << endl;
25 1903 : const char *partNames[] = {"actualDurations", "actualTimes", "autoData", "crossData", "zeroLags", "flags"};
26 1903 : set<string> dummy(partNames, partNames+6);
27 1903 : s_partNames = dummy;
28 1903 : sdmDataSubset = SDMDataSubset(&sdmDataObject);
29 :
30 1903 : boundary_1 = "" ;
31 1903 : currentState = S_NO_BDF;
32 : // cout << "SDMDataObjectStreamReader::SDMDataObjectStreamReader() : exiting" << endl;
33 1903 : }
34 :
35 1903 : SDMDataObjectStreamReader::~SDMDataObjectStreamReader() {
36 : //cout << "SDMDataObjectStreamReader::~SDMDataObjectStreamReader : entering" << endl;
37 1903 : this->close();
38 : //cout << "SDMDataObjectStreamReader::~SDMDataObjectStreamReader : exiting" << endl;
39 1903 : }
40 :
41 2515 : void SDMDataObjectStreamReader::open(const string& path) {
42 2515 : checkState(T_OPEN, "open");
43 2515 : this->path = path;
44 2515 : f.open(path.c_str(), ifstream::in);
45 2515 : if (f.fail())
46 0 : throw SDMDataObjectStreamReaderException("could not open '" + path + "'.");
47 :
48 2515 : boundary_1 = requireMIMEHeader();
49 : // cout << "Boundary = " << boundary_1 << endl;
50 2515 : sdmDataObject.valid_ = true;
51 2515 : requireSDMDataHeaderMIMEPart();
52 2515 : sdmDataObject.owns();
53 2515 : integrationIndex = -1; // We have not yet read any integration.
54 :
55 2515 : currentState = S_AT_BEGINNING;
56 2515 : }
57 :
58 40548 : int64_t SDMDataObjectStreamReader::position() {
59 40548 : return (int64_t) f.tellg();
60 : }
61 :
62 40548 : void SDMDataObjectStreamReader::position(int64_t p) {
63 40548 : f.seekg(p);
64 40548 : }
65 :
66 4742 : void SDMDataObjectStreamReader::close() {
67 : // cout << "SDMDataObjectStreamReader::close -- Entering" << endl;
68 4742 : if (f.is_open()) {
69 2515 : releaseMemory(sdmDataSubset);
70 2515 : for (unsigned int i = 0; i < remainingSubsets.size(); i++)
71 0 : releaseMemory(remainingSubsets[i]);
72 22235 : for (unsigned int i = 0; i < someSubsets.size(); i++)
73 19720 : releaseMemory(someSubsets[i]);
74 2515 : f.close();
75 2515 : currentState = S_NO_BDF;
76 : }
77 : // cout << "SDMDataObjectStreamReader::close -- Exiting" << endl;
78 4742 : }
79 :
80 0 : unsigned int SDMDataObjectStreamReader::currentIntegrationIndex() const { checkState(T_QUERY, "currentIntegrationIndex"); return integrationIndex; }
81 0 : unsigned long long SDMDataObjectStreamReader::currentIntegrationStartsAt() const { checkState(T_QUERY, "currentIntegrationStartAt"); return integrationStartsAt; }
82 0 : string SDMDataObjectStreamReader::title() const { checkState(T_QUERY, "title"); return sdmDataObject.title(); }
83 29 : const ByteOrder* SDMDataObjectStreamReader::byteOrder() const { checkState(T_QUERY, "byteOrder"); return sdmDataObject.byteOrder(); }
84 0 : unsigned long long SDMDataObjectStreamReader::startTime() const { checkState(T_QUERY, "startTime"); return sdmDataObject.startTime(); }
85 4622 : unsigned int SDMDataObjectStreamReader::numTime() const { checkState(T_QUERY, "numTime"); return sdmDataObject.numTime(); }
86 0 : string SDMDataObjectStreamReader::dataOID() const { checkState(T_QUERY, "dataOID"); return sdmDataObject.dataOID(); }
87 0 : string SDMDataObjectStreamReader::execBlockUID() const { checkState(T_QUERY, "execBlockUID"); return sdmDataObject.execBlockUID(); }
88 0 : unsigned int SDMDataObjectStreamReader::execBlockNum() const { checkState(T_QUERY, "execBlockNum"); return sdmDataObject.execBlockNum(); }
89 0 : unsigned int SDMDataObjectStreamReader::scanNum() const { checkState(T_QUERY, "scanNum"); return sdmDataObject.scanNum(); }
90 0 : unsigned int SDMDataObjectStreamReader::subscanNum() const { checkState(T_QUERY, "subscanNum"); return sdmDataObject.subscanNum(); }
91 1760 : unsigned int SDMDataObjectStreamReader::numAntenna() const { checkState(T_QUERY, "numAntenna"); return sdmDataObject.numAntenna(); }
92 2492 : CorrelationMode SDMDataObjectStreamReader::correlationMode() const { checkState(T_QUERY, "correlationMode"); return sdmDataObject.correlationMode(); }
93 0 : OptionalSpectralResolutionType SDMDataObjectStreamReader::spectralResolutionType() const { checkState(T_QUERY, "spectralResolutionType"); return sdmDataObject.spectralResolutionType(); }
94 1760 : ProcessorType SDMDataObjectStreamReader::processorType() const { checkState(T_QUERY, "processorType"); return sdmDataObject.processorType(); }
95 0 : CorrelatorType SDMDataObjectStreamReader::correlatorType() const { checkState(T_QUERY, "correlatorType"); return sdmDataObject.correlatorType(); }
96 1063 : bool SDMDataObjectStreamReader::hasPackedData() const { checkState(T_QUERY, "isTP"); return sdmDataObject.hasPackedData();}
97 0 : bool SDMDataObjectStreamReader::isTP() const { checkState(T_QUERY, "isTP"); return sdmDataObject.isTP();}
98 0 : bool SDMDataObjectStreamReader::isWVR() const {checkState(T_QUERY, "isCorrelation"); return sdmDataObject.isWVR();}
99 0 : bool SDMDataObjectStreamReader::isCorrelation() const {checkState(T_QUERY, "isCorrelation"); return sdmDataObject.isCorrelation();}
100 11525 : const SDMDataObject::DataStruct& SDMDataObjectStreamReader::dataStruct() const { checkState(T_QUERY, "dataStruct"); return sdmDataObject.dataStruct();}
101 0 : bool SDMDataObjectStreamReader::aborted() const {checkState(T_QUERY, "aborted"); return sdmDataObject.aborted();}
102 0 : unsigned long long SDMDataObjectStreamReader::abortTime() const { checkState(T_QUERY, "abortTime"); return sdmDataObject.abortTime(); }
103 0 : string SDMDataObjectStreamReader::abortReason() const { checkState(T_QUERY, "abortReason"); return sdmDataObject.abortReason(); }
104 0 : string SDMDataObjectStreamReader::toString() const { checkState(T_QUERY, "toString"); return sdmDataObject.toString(); }
105 :
106 88754 : void SDMDataObjectStreamReader::checkState(Transitions t, const string& methodName) const {
107 : // cout << "Entering checkState with currentState = " << currentState << " and transition = " << t << " for method " << methodName << endl;
108 88754 : switch (currentState) {
109 2515 : case S_NO_BDF:
110 2515 : if ( t == T_OPEN ) {
111 2515 : return;
112 : }
113 0 : break;
114 :
115 32579 : case S_AT_BEGINNING :
116 32579 : switch (t) {
117 32579 : case T_QUERY:
118 : case T_TEST_END:
119 : case T_READ:
120 : case T_READ_NEXT:
121 : case T_READ_ALL:
122 : case T_CLOSE:
123 32579 : return;
124 0 : default :
125 0 : break;
126 : }
127 0 : break;
128 :
129 53660 : case S_READING:
130 53660 : switch (t) {
131 53660 : case T_TEST_END:
132 : case T_READ:
133 : case T_READ_NEXT:
134 : case T_READ_ALL:
135 : case T_QUERY:
136 : case T_CLOSE:
137 53660 : return;
138 0 : default :
139 0 : break;
140 : }
141 0 : break;
142 :
143 0 : case S_AT_END:
144 0 : switch(t) {
145 0 : case T_TEST_END:
146 : case T_QUERY:
147 : case T_READ_NEXT:
148 : case T_READ_ALL:
149 : case T_CLOSE:
150 0 : return;
151 0 : default:
152 0 : break;
153 : }
154 0 : break;
155 : } // end switch on currentState
156 :
157 : // Any other pair (transition, currentState) will throw an exception.
158 0 : throw SDMDataObjectStreamReaderException("Invalid call of method '" + methodName + "' in the current context.");
159 : }
160 :
161 39693 : bool SDMDataObjectStreamReader::hasSubset() {
162 39693 : checkState(T_TEST_END, "hasSubset");
163 39693 : bool atEnd = currentLine.compare("--"+boundary_1+"--") == 0;
164 39693 : if (atEnd) currentState = S_AT_END;
165 39693 : return !atEnd;
166 : }
167 :
168 535 : const vector<SDMDataSubset>& SDMDataObjectStreamReader::nextSubsets(unsigned int nSubsets) {
169 535 : checkState(T_READ_NEXT, "nextSubsets");
170 :
171 : // Deep empty of the vector nextSubsets
172 : // Firstly free all memory dynamically allocated in every element of the vector.
173 16916 : for (unsigned int i = 0; i < someSubsets.size(); i++)
174 16381 : releaseMemory(someSubsets.at(i));
175 :
176 : // Then clear the vector.
177 535 : someSubsets.clear();
178 :
179 : // Then populate the vector nextSubsets with as many SDMDataSubsets as possible up to a limit
180 : // of nSubsets read from the current position.
181 535 : unsigned int nRead = 0;
182 18323 : while ((nRead < nSubsets) && hasSubset()) {
183 17788 : someSubsets.push_back(SDMDataSubset(&sdmDataObject));
184 17788 : integrationIndex++; nRead++;
185 17788 : requireSDMDataSubsetMIMEPart(someSubsets.back());
186 17788 : string line = nextLine();
187 : }
188 :
189 535 : return someSubsets;
190 : }
191 :
192 0 : const vector<SDMDataSubset>& SDMDataObjectStreamReader::allRemainingSubsets() {
193 : // cout << "SDMDataObjectStreamReader::allRemainingSubsets: entering." << endl;
194 0 : checkState(T_READ_ALL, "allRemainingSubsets");
195 :
196 : // Deep empty of the vector remainingSubsets.
197 : // Firstly free all memory dynamically allocated in every element of the vector.
198 0 : for (unsigned int i = 0; i < remainingSubsets.size(); i++)
199 0 : releaseMemory(remainingSubsets.at(i));
200 :
201 : // Then clear the vector.
202 0 : remainingSubsets.clear();
203 :
204 : // Then populate the vector with a new collection.
205 0 : while (hasSubset()) {
206 0 : remainingSubsets.push_back(SDMDataSubset(&sdmDataObject));
207 0 : integrationIndex++;
208 0 : requireSDMDataSubsetMIMEPart(remainingSubsets.back());
209 0 : string line = nextLine();
210 : }
211 :
212 : // cout << "SDMDataObjectStreamReader::allRemainingSubsets: exiting." << endl;
213 0 : return remainingSubsets;
214 : }
215 :
216 22760 : const SDMDataSubset & SDMDataObjectStreamReader::getSubset() {
217 22760 : checkState(T_READ, "getSubset");
218 22760 : integrationIndex++;
219 22760 : requireSDMDataSubsetMIMEPart(sdmDataSubset);
220 22760 : string line = nextLine();
221 :
222 22760 : currentState = S_READING;
223 45520 : return sdmDataSubset;
224 : }
225 :
226 961259 : string SDMDataObjectStreamReader::nextLine() {
227 961259 : unsigned long long whereAmI = f.tellg();
228 961259 : getline(f, currentLine);
229 961259 : if (f.fail()) {
230 0 : ostringstream oss ;
231 0 : oss << "SDMDataObjectStreamReader::nextLine() : I could not read a line in '" << path << "' at position " << whereAmI << ".";
232 0 : throw SDMDataObjectStreamReaderException(oss.str());
233 : }
234 : // cout << "nextLine has read '" << currentLine << "'" << endl;
235 961259 : return currentLine;
236 : }
237 :
238 291712 : pair<string, string> SDMDataObjectStreamReader::headerField2Pair(const string& hf){
239 583424 : string name, value;
240 291712 : size_t colonIndex = hf.find(":");
241 291712 : if (colonIndex == string::npos)
242 0 : throw SDMDataObjectStreamReaderException(" could not detect a well formed MIME header field in '"+hf+"'");
243 :
244 291712 : if (colonIndex > 0) {
245 291712 : name = hf.substr(0, colonIndex);
246 : #ifndef WITHOUT_BOOST
247 : boost::algorithm::trim(name);
248 : #else
249 291712 : asdm::trim(name);
250 : #endif
251 : }
252 :
253 291712 : if (colonIndex < hf.size()) {
254 291712 : value = hf.substr(colonIndex+1);
255 : #ifndef WITHOUT_BOOST
256 : boost::algorithm::trim(value);
257 : #else
258 291712 : asdm::trim(value);
259 : #endif
260 : }
261 :
262 583424 : return make_pair(name, value);
263 : }
264 :
265 289197 : pair<string, string> SDMDataObjectStreamReader::requireHeaderField(const string & hf) {
266 289197 : pair<string, string> hf2pair(headerField2Pair(nextLine()));
267 : // cout << hf2pair.first << ", " << hf2pair.second << endl;
268 : #ifndef WITHOUT_BOOST
269 : if (boost::algorithm::to_upper_copy(hf2pair.first) != hf)
270 : throw SDMDataObjectStreamReaderException("read '" + currentLine + "'. Was expecting '" + hf + "'...");
271 : #else
272 289197 : if (asdm::str_toupper(hf2pair.first) != hf)
273 0 : throw SDMDataObjectStreamReaderException("read '" + currentLine + "'. Was expecting '" + hf + "'...");
274 : #endif
275 289197 : return hf2pair;
276 : }
277 :
278 43063 : string unquote(const string& s, string& unquoted) {
279 43063 : if (s.size() >= 2)
280 43063 : if (((s.at(0) == '"') && (s.at(s.size()-1) == '"')) || ((s.at(0) == '\'') && (s.at(s.size()-1) == '\''))) {
281 41628 : if (s.size() == 2)
282 0 : unquoted = "";
283 : else
284 41628 : unquoted = s.substr(1, s.size() - 2);
285 : }
286 : else
287 1435 : unquoted = s;
288 : else
289 0 : unquoted = s;
290 43063 : return unquoted;
291 : }
292 :
293 :
294 :
295 43063 : string SDMDataObjectStreamReader::requireBoundaryInCT(const string& ctValue) {
296 86126 : vector<string> cvValueItems;
297 :
298 : #ifndef WITHOUT_BOOST
299 : boost::algorithm::split (cvValueItems, ctValue, boost::algorithm::is_any_of(";"));
300 : vector<string> cvValueItemsNameValue;
301 : for ( vector<string>::const_iterator iter = cvValueItems.begin(); iter != cvValueItems.end() ; iter++ ) {
302 : cvValueItemsNameValue.clear();
303 : boost::algorithm::split(cvValueItemsNameValue, *iter, boost::algorithm::is_any_of("="));
304 : string boundary;
305 : if ((cvValueItemsNameValue.size() > 1) && (boost::algorithm::to_upper_copy(boost::algorithm::trim_copy(cvValueItemsNameValue[0])) == "BOUNDARY") && (unquote(cvValueItemsNameValue[1], boundary).size() > 0))
306 : return boundary;
307 : }
308 : #else
309 43063 : asdm::strsplit (ctValue, ';', cvValueItems);
310 86126 : vector<string> cvValueItemsNameValue;
311 86126 : for ( vector<string>::const_iterator iter = cvValueItems.begin(); iter != cvValueItems.end() ; iter++ ) {
312 86126 : cvValueItemsNameValue.clear();
313 86126 : asdm::strsplit(*iter,'=',cvValueItemsNameValue);
314 86126 : string boundary;
315 86126 : if ((cvValueItemsNameValue.size() > 1) && (asdm::str_toupper(asdm::trim_copy(cvValueItemsNameValue[0])) == "BOUNDARY") && (unquote(cvValueItemsNameValue[1], boundary).size() > 0))
316 86126 : return boundary;
317 : }
318 : #endif
319 0 : throw SDMDataObjectStreamReaderException("could not find a boundary definition in '" + ctValue + "'.");
320 : }
321 :
322 40548 : void SDMDataObjectStreamReader::skipAsLongAsLineStartsWith(const string& start) {
323 40548 : int64_t curpos = 0;
324 0 : do {
325 40548 : curpos = position();
326 40548 : nextLine();
327 : }
328 40548 : while (currentLine.find(start) == 0);
329 40548 : position(curpos);
330 40548 : return;
331 : }
332 :
333 145856 : void SDMDataObjectStreamReader::skipUntilEmptyLine(int maxSkips) {
334 : // cout << "Entering skipUntilEmptyLine" << endl;
335 145856 : int numSkips = 0;
336 291712 : string line = nextLine();
337 238566 : while ((line.size() != 0) && (numSkips <= maxSkips)) {
338 92710 : line = nextLine();
339 92710 : numSkips += 1;
340 : }
341 :
342 145856 : if (numSkips > maxSkips) {
343 0 : ostringstream oss;
344 0 : oss << "could not find an empty line is less than " << maxSkips + 1 << " lines." << endl;
345 0 : throw SDMDataObjectStreamReaderException(oss.str());
346 : }
347 : // cout << "Exiting skipUntilEmptyLine" << endl;
348 145856 : }
349 :
350 :
351 2515 : string SDMDataObjectStreamReader::requireMIMEHeader() {
352 : // MIME-Version
353 5030 : pair<string, string>name_value(headerField2Pair(nextLine()));
354 : // cout << name_value.first << "=" << name_value.second << endl;
355 : // if (currentLine != "MIME-Version: 1.0") // a work around for the case when the very first character is not the expected "M" (happened with some corrupted data).
356 : #ifndef WITHOUT_BOOST
357 : if (! boost::algorithm::iends_with(currentLine, "IME-Version: 1.0"))
358 : #else
359 5030 : std::string versionEnd = "IME-Version: 1.0";
360 2515 : if ((currentLine.size()<=versionEnd.size()) || (currentLine.compare((currentLine.size()-versionEnd.size()),versionEnd.size(),versionEnd)!=0))
361 : #endif
362 0 : throw SDMDataObjectStreamReaderException("'MIME-Version: 1.0' missing at the very beginning of the file '"+path+"'.");
363 :
364 : // Content-Type
365 2515 : boundary_1 = requireBoundaryInCT(requireHeaderField("CONTENT-TYPE").second);
366 :
367 : // cout << "boundary_1 =" << boundary_1 << endl;
368 :
369 : // Content-Description
370 2515 : name_value = requireHeaderField("CONTENT-DESCRIPTION");
371 :
372 : // Content-Location
373 2515 : name_value = requireHeaderField("CONTENT-LOCATION");
374 :
375 : // Look for an empty line in the at most 10 subsequent lines.
376 2515 : skipUntilEmptyLine(10);
377 :
378 5030 : return boundary_1;
379 : }
380 :
381 43063 : string SDMDataObjectStreamReader::accumulateUntilBoundary(const string& boundary, int maxLines) {
382 : // cout << "Entering accumulateUntilBoundary with maxLines = " << maxLines << endl;
383 43063 : int numLines = 0;
384 86126 : string line = nextLine();
385 43063 : string result;
386 104890 : while ((numLines <= maxLines) && (line.find("--"+boundary) == string::npos)) {
387 61827 : result += line;
388 61827 : numLines++;
389 61827 : line = nextLine();
390 : }
391 :
392 43063 : if (numLines > maxLines) {
393 0 : ostringstream oss;
394 0 : oss << "could not find the boundary string '"<< boundary << "' in less than " << maxLines + 1 << " lines." << endl;
395 0 : throw SDMDataObjectStreamReaderException(oss.str());
396 : }
397 :
398 86126 : return result;
399 : }
400 :
401 43063 : void SDMDataObjectStreamReader::requireBoundary(const string& boundary, int maxLines) {
402 : // cout << "Entering require boundary with boundary == '" << boundary << "' and maxLines = " << maxLines << endl;
403 43063 : int numLines = 0;
404 86126 : string dashdashBoundary = "--"+boundary;
405 86126 : string line = nextLine();
406 44439 : while ((numLines <= maxLines) && (line.compare(dashdashBoundary) != 0)) {
407 1376 : numLines++;
408 1376 : line = nextLine();
409 : }
410 :
411 43063 : if (numLines > maxLines) {
412 0 : ostringstream oss;
413 0 : oss << "could not find the boundary string '"<< boundary << "' in less than " << maxLines + 1 << " lines." << endl;
414 0 : throw SDMDataObjectStreamReaderException(oss.str());
415 : }
416 43063 : }
417 :
418 0 : void SDMDataObjectStreamReader::lookForBinaryPartSize(xmlNode* aNode) {
419 : #ifndef WITHOUT_BOOST
420 : const boost::regex UINT("[0-9]+");
421 : #else
422 0 : const std::regex UINT("[0-9]+");
423 : #endif
424 0 : xmlNode *curNode = NULL;
425 :
426 0 : for (curNode = aNode; curNode ; curNode = curNode->next) {
427 0 : if (curNode->type == XML_ELEMENT_NODE) {
428 0 : if (s_partNames.find(string((char *)curNode->name)) != s_partNames.end()){
429 0 : if (xmlHasProp(curNode, (const xmlChar*) "size")) {
430 0 : xmlChar * value = xmlGetProp(curNode, (const xmlChar *) "size");
431 : #ifndef WITHOUT_BOOST
432 : boost::cmatch what;
433 : if (boost::regex_match((char*) value, what, UINT)) {
434 : #else
435 0 : std::cmatch what;
436 0 : if (std::regex_match((char*) value, what, UINT)) {
437 : #endif
438 0 : int64_t result = ::atoi(what[0].first);
439 0 : xmlFree(value);
440 0 : binaryPartSize[string((char *) curNode->name)] = result;
441 : }
442 : else {
443 0 : xmlFree(value);
444 0 : throw SDMDataObjectStreamReaderException("In '" + string((const char*) curNode->name) + "' failed to parse the value of '"+string((const char*) value)+"' as an int.");
445 : }
446 : }
447 : else {
448 0 : throw SDMDataObjectStreamReaderException("In '" + string((const char*) curNode->name) + "' could not find the attribute 'size'.");
449 : }
450 : }
451 : }
452 0 : lookForBinaryPartSize(curNode->children);
453 : }
454 0 : }
455 :
456 0 : string SDMDataObjectStreamReader::requireCrossDataType(xmlNode* parent) {
457 0 : string result;
458 :
459 0 : string comparee("crossData");
460 0 : xmlNode * child = parent->children;
461 :
462 0 : while ((child != 0) && (comparee.compare((const char*) child->name) != 0))
463 0 : child = child->next;
464 :
465 0 : if ((child == 0) || (child->type != XML_ELEMENT_NODE)) {
466 0 : ostringstream oss;
467 0 : oss << "could not find the element 'crossData'." << endl;
468 0 : throw SDMDataObjectStreamReaderException(oss.str());
469 : }
470 :
471 0 : if (xmlHasProp(child, (const xmlChar*) "type")) {
472 0 : xmlChar * value = xmlGetProp(child, (const xmlChar *) "type");
473 0 : result = string((const char *) value);
474 0 : xmlFree(value);
475 0 : return result;
476 : }
477 : else
478 0 : throw SDMDataObjectStreamReaderException("In '" + string((const char*) child->name) + "' could not find the attribute 'type'.");
479 : }
480 :
481 0 : void SDMDataObjectStreamReader::printElementNames(xmlNode * a_node) {
482 0 : xmlNode *cur_node = NULL;
483 :
484 0 : for (cur_node = a_node; cur_node; cur_node = cur_node->next) {
485 0 : if (cur_node->type == XML_ELEMENT_NODE) {
486 0 : cout << "node type: Element, name: " << cur_node->name << endl;
487 : }
488 0 : printElementNames(cur_node->children);
489 : }
490 0 : }
491 :
492 2515 : void SDMDataObjectStreamReader::requireSDMDataHeaderMIMEPart() {
493 : //
494 : // Requires the presense of boundary_1
495 2515 : requireBoundary(boundary_1, 0);
496 :
497 : // Ignore header fields
498 : // requireHeaderField("CONTENT-TYPE")
499 : // requireHeaderField("CONTENT-TRANSFER-ENCODING")
500 : // requireHeaderField("CONTENT-LOCATION")
501 :
502 : //
503 : // Look for an empty line at most distant from 100 lines from here.
504 2515 : skipUntilEmptyLine(10);
505 5030 : string sdmDataHeader = accumulateUntilBoundary(boundary_1, 100);
506 :
507 : /*
508 : xmlDoc * doc = xmlReadMemory(sdmDataHeader.data(), sdmDataHeader.size(), "SDMDataHeader.xml", NULL, XML_PARSE_NOBLANKS);
509 : xmlChar* xmlBuff;
510 : int bufferSize;
511 : xmlDocDumpFormatMemory(doc, &xmlBuff, &bufferSize, 1);
512 : cout << string((const char*) xmlBuff, bufferSize) << endl;
513 : xmlFree(xmlBuff);
514 : xmlNode* root_element = xmlDocGetRootElement(doc);
515 : if ( root_element == NULL || root_element->type != XML_ELEMENT_NODE )
516 : throw SDMDataObjectStreamReaderException("Failed to parse the SDMDataHeader into a DOM structure.");
517 : */
518 :
519 2515 : parser.parseMemoryHeader(sdmDataHeader, sdmDataObject);
520 : unsigned int bps;
521 2515 : if ((bps = sdmDataObject.dataStruct().flags().size()))
522 2391 : binaryPartSize["flags"] = bps;
523 :
524 2515 : if ((bps = sdmDataObject.dataStruct().actualTimes().size()))
525 1946 : binaryPartSize["actualTimes"] = bps;
526 :
527 2515 : if ((bps = sdmDataObject.dataStruct().actualDurations().size()))
528 1946 : binaryPartSize["actualDurations"] = bps;
529 :
530 2515 : if ((bps = sdmDataObject.dataStruct().zeroLags().size()))
531 414 : binaryPartSize["zeroLags"] = bps;
532 :
533 2515 : if ((bps = sdmDataObject.dataStruct().autoData().size()))
534 2441 : binaryPartSize["autoData"] = bps;
535 :
536 2515 : if ((bps = sdmDataObject.dataStruct().crossData().size()))
537 756 : binaryPartSize["crossData"] = bps;
538 2515 : }
539 :
540 40548 : void SDMDataObjectStreamReader::requireSDMDataSubsetMIMEPart(SDMDataSubset & sdmDataSubset) {
541 40548 : integrationStartsAt = f.tellg();
542 :
543 40548 : skipAsLongAsLineStartsWith("--"+boundary_1); // <--- this was added to preserve compatiblity with SDMDataObjectReader, which
544 : // was less strict on the respect of the BDF specifications. (cf CAS-8151)
545 : // Here the problem was caused by two successive occurrences of --MIME_boundary_1
546 : // instead of only one as indicated in the specs at the very beginning of
547 : // an SDMDataSubsetHeader. M.Caillat - 4 decembre 2015
548 121644 : pair<string, string> name_value = requireHeaderField("CONTENT-TYPE");
549 40548 : boundary_2 = requireBoundaryInCT(name_value.second);
550 : // cout << "boundary_2 = " << boundary_2 << endl;
551 40548 : name_value = requireHeaderField("CONTENT-DESCRIPTION");
552 40548 : requireBoundary(boundary_2, 10);
553 40548 : skipUntilEmptyLine(10);
554 :
555 : //
556 : // We assume that the subset header can't be longer than 100 lines.
557 : //
558 81096 : string sdmDataSubsetHeader = accumulateUntilBoundary(boundary_2, 100);
559 :
560 : //
561 : // Empty sdmDataSubset if necessary.
562 : //
563 40548 : releaseMemory(sdmDataSubset);
564 :
565 : // We can start to acquire a new SDMDataSubset.
566 40548 : if (sdmDataObject.isCorrelation())
567 40053 : parser.parseMemoryCorrSubsetHeader(sdmDataSubsetHeader, sdmDataSubset);
568 : else
569 495 : parser.parseMemoryTPSubsetHeader(sdmDataSubsetHeader, sdmDataSubset);
570 :
571 40548 : attachmentFlags.reset();
572 : #ifndef WITHOUT_BOOST
573 : boost::regex BINARYPARTLOC("([0-9]+/)+(actualDurations|actualTimes|autoData|crossData|zeroLags|flags)\\.bin");
574 : #else
575 81096 : std::regex BINARYPARTLOC("([0-9]+/)+(actualDurations|actualTimes|autoData|crossData|zeroLags|flags)\\.bin");
576 : #endif
577 40548 : bool done = false;
578 140826 : while (!done) {
579 100278 : name_value = requireHeaderField("CONTENT-TYPE");
580 100278 : name_value = requireHeaderField("CONTENT-LOCATION");
581 :
582 : #ifndef WITHOUT_BOOST
583 : boost::smatch what;
584 : const string contentLocation = boost::algorithm::trim_copy(name_value.second);
585 : if (!boost::regex_search(contentLocation, what, BINARYPARTLOC)) {
586 : throw SDMDataObjectStreamReaderException("Invalid field : '" + name_value.first + ":" + name_value.second + "'.");
587 : }
588 : #else
589 200556 : std::smatch what;
590 200556 : const string contentLocation = asdm::trim_copy(name_value.second);
591 100278 : if (!std::regex_search(contentLocation, what, BINARYPARTLOC)) {
592 0 : throw SDMDataObjectStreamReaderException("Invalid field : '" + name_value.first + ":" + name_value.second + "'.");
593 : }
594 : #endif
595 : // cout << "Binary part name = " << what[2] << "...";
596 200556 : string binaryPartName = string(what[2]);
597 100278 : if (binaryPartSize.find(binaryPartName) == binaryPartSize.end())
598 0 : throw SDMDataObjectStreamReaderException("The size of '"+binaryPartName+"' was not announced in the data header.!");
599 :
600 100278 : if (binaryPartSize[binaryPartName] == 0)
601 0 : throw SDMDataObjectStreamReaderException("The size of '"+binaryPartName+"' was announced as null. I was not expecting a '"+binaryPartName+"' attachment here.");
602 :
603 100278 : skipUntilEmptyLine(10);
604 100278 : int numberOfCharsPerValue = 0;
605 100278 : char** binaryPartPtrPtr = 0;
606 100278 : if (binaryPartName == "actualDurations") {
607 1468 : attachmentFlags.set(ACTUALDURATIONS);
608 1468 : binaryPartPtrPtr = (char**) &sdmDataSubset.actualDurations_;
609 1468 : sdmDataSubset.nActualDurations_ = binaryPartSize[binaryPartName];
610 1468 : sdmDataSubset.actualDurationsPosition_ = f.tellg();
611 1468 : numberOfCharsPerValue = 8;
612 : }
613 98810 : else if (binaryPartName == "actualTimes") {
614 1468 : attachmentFlags.set(ACTUALTIMES);
615 1468 : binaryPartPtrPtr = (char **) &sdmDataSubset.actualTimes_;
616 1468 : sdmDataSubset.nActualTimes_ = binaryPartSize[binaryPartName];
617 1468 : sdmDataSubset.actualTimesPosition_ = f.tellg();
618 1468 : numberOfCharsPerValue = 8;
619 : }
620 97342 : else if (binaryPartName == "autoData") {
621 40436 : attachmentFlags.set(AUTODATA);
622 40436 : binaryPartPtrPtr = (char **) &sdmDataSubset.autoData_;
623 40436 : sdmDataSubset.nAutoData_ = binaryPartSize[binaryPartName];
624 40436 : sdmDataSubset.autoDataPosition_ = f.tellg();
625 40436 : numberOfCharsPerValue = 4;
626 : }
627 56906 : else if (binaryPartName == "crossData") {
628 26683 : attachmentFlags.set(CROSSDATA);
629 26683 : sdmDataSubset.shortCrossData_ = 0;
630 26683 : sdmDataSubset.longCrossData_ = 0;
631 26683 : sdmDataSubset.floatCrossData_ = 0;
632 26683 : sdmDataSubset.nCrossData_ = binaryPartSize[binaryPartName];
633 26683 : sdmDataSubset.crossDataPosition_ = f.tellg();
634 26683 : PrimitiveDataType pdt = sdmDataSubset.crossDataType();
635 26683 : switch (pdt) {
636 22543 : case INT16_TYPE:
637 22543 : binaryPartPtrPtr = (char **) &sdmDataSubset.shortCrossData_;
638 22543 : numberOfCharsPerValue = 2;
639 22543 : break;
640 2652 : case INT32_TYPE:
641 2652 : binaryPartPtrPtr = (char **) &sdmDataSubset.longCrossData_;
642 2652 : numberOfCharsPerValue = 4;
643 2652 : break;
644 1488 : case FLOAT32_TYPE:
645 1488 : binaryPartPtrPtr = (char **) &sdmDataSubset.floatCrossData_;
646 1488 : numberOfCharsPerValue = 4;
647 1488 : break;
648 0 : default:
649 0 : throw SDMDataObjectStreamReaderException("Invalid data type for cross data '" + CPrimitiveDataType::name(pdt) + "'.");
650 : }
651 : }
652 30223 : else if (binaryPartName == "flags") {
653 8976 : attachmentFlags.set(FLAGS);
654 8976 : binaryPartPtrPtr = (char **) &sdmDataSubset.flags_;
655 8976 : sdmDataSubset.nFlags_ = binaryPartSize[binaryPartName];
656 8976 : sdmDataSubset.flagsPosition_ = f.tellg();
657 8976 : numberOfCharsPerValue = 4;
658 : }
659 21247 : else if (binaryPartName == "zeroLags") {
660 21247 : attachmentFlags.set(ZEROLAGS);
661 21247 : binaryPartPtrPtr = (char **) &sdmDataSubset.zeroLags_;
662 21247 : sdmDataSubset.nZeroLags_ = binaryPartSize[binaryPartName];
663 21247 : sdmDataSubset.zeroLagsPosition_ = f.tellg();
664 21247 : numberOfCharsPerValue = 4;
665 : }
666 :
667 100278 : int64_t numberOfCharsToRead = numberOfCharsPerValue * binaryPartSize[binaryPartName];
668 100278 : *binaryPartPtrPtr = new char[numberOfCharsToRead * sizeof(char)];
669 100278 : if (*binaryPartPtrPtr == 0) {
670 0 : ostringstream oss;
671 0 : oss << "Processing integration # " << integrationIndex << ": I could not get memory to store '" << binaryPartName << "'." << endl;
672 0 : throw SDMDataObjectStreamReaderException(oss.str());
673 : }
674 :
675 100278 : f.read(*binaryPartPtrPtr, numberOfCharsToRead * sizeof(char));
676 100278 : if (f.fail()) {
677 0 : ostringstream oss;
678 0 : oss << "Processing integration # " << integrationIndex << ": a problem occurred while reading '" << binaryPartName << "'." << endl;
679 0 : throw SDMDataObjectStreamReaderException(oss.str());
680 : }
681 100278 : if (f.eof()) {
682 0 : ostringstream oss;
683 0 : oss << "Processing integration # " << integrationIndex << ": a unexpected end of file occurred while reading '" << binaryPartName << "'." << endl;
684 0 : throw SDMDataObjectStreamReaderException(oss.str());
685 : }
686 :
687 100278 : string line = nextLine(); // Absorb the nl right after the last byte of the binary attachment
688 100278 : line = nextLine(); // This should boundary_2
689 :
690 100278 : if (line.find("--" + boundary_2) != 0) {
691 0 : ostringstream oss;
692 0 : oss << "Processing integration # " << integrationIndex << ": unexpected '" << line << "' after the binary part '" << binaryPartName << "'." << endl;
693 0 : throw SDMDataObjectStreamReaderException(oss.str());
694 : }
695 :
696 100278 : done = line.compare("--" + boundary_2+"--") == 0;
697 : }
698 :
699 : // Now check if the binary attachments found are compatible with the correlation mode
700 : // and if their sizes are equal to what is announced in the global header.
701 : //
702 : // The presence of crossData and autoData depends on the correlation mode.
703 :
704 40548 : switch (sdmDataObject.correlationMode()) {
705 112 : case CROSS_ONLY:
706 112 : if (!attachmentFlags.test(CROSSDATA)) {
707 0 : ostringstream oss;
708 0 : oss << "Data subset '"<<sdmDataSubset.projectPath()<<"': ";
709 0 : oss << "a binary attachment 'crossData' was expected in integration #" << integrationIndex;
710 0 : throw SDMDataObjectStreamReaderException(oss.str());
711 : }
712 :
713 112 : if (attachmentFlags.test(AUTODATA)) {
714 0 : ostringstream oss;
715 0 : oss << "Data subset '"<<sdmDataSubset.projectPath()<<"': ";
716 0 : oss << "found an unexpected attachment 'autoData' in integration #" << integrationIndex << ".";
717 0 : throw SDMDataObjectStreamReaderException(oss.str());
718 : }
719 112 : break;
720 :
721 13865 : case AUTO_ONLY:
722 13865 : if (!attachmentFlags.test(AUTODATA)) {
723 0 : ostringstream oss;
724 0 : oss << "Data subset '"<<sdmDataSubset.projectPath()<<"': ";
725 0 : oss << "a binary attachment 'autoData' was expected.in integration #" << integrationIndex << ".";
726 0 : throw SDMDataObjectStreamReaderException(oss.str());
727 : }
728 :
729 13865 : if (attachmentFlags.test(CROSSDATA)) {
730 0 : ostringstream oss;
731 0 : oss << "Data subset '"<<sdmDataSubset.projectPath()<<"': ";
732 0 : oss << "found an unexpected attachment 'crossData' in integration #" << integrationIndex << ".";
733 0 : throw SDMDataObjectStreamReaderException(oss.str());
734 : }
735 13865 : break;
736 :
737 26571 : case CROSS_AND_AUTO:
738 26571 : if (!attachmentFlags.test(AUTODATA)) {
739 0 : ostringstream oss;
740 0 : oss << "Data subset '"<<sdmDataSubset.projectPath()<<"': ";
741 0 : oss << "a binary attachment 'autoData' was expected in integration #" << integrationIndex << ".";
742 0 : throw SDMDataObjectStreamReaderException(oss.str());
743 : }
744 :
745 26571 : if (!attachmentFlags.test(CROSSDATA)) {
746 0 : ostringstream oss;
747 0 : oss << "Data subset '"<<sdmDataSubset.projectPath()<<"': ";
748 0 : oss << "a binary attachment 'crossData' was expected in integration #" << integrationIndex << ".";
749 0 : throw SDMDataObjectStreamReaderException(oss.str());
750 : }
751 26571 : break;
752 :
753 0 : default:
754 0 : throw SDMDataObjectStreamReaderException("Data subset '"+sdmDataSubset.projectPath()+"': unrecognized correlation mode");
755 : break;
756 : }
757 :
758 :
759 40548 : if (attachmentFlags.test(ZEROLAGS)) {
760 : // Refuse the zeroLags attachment if it's not a Correlator or if the correlator is a CORRELATOR_FX (ACA).
761 21247 : if ((sdmDataObject.processorType_ != CORRELATOR) || (sdmDataObject.correlatorType() == FX))
762 0 : throw SDMDataObjectStreamReaderException("zeroLags are not expected from a correlator CORRELATOR_FX");
763 : }
764 40548 : }
765 :
766 79164 : void SDMDataObjectStreamReader::releaseMemory(SDMDataSubset & sdmDataSubset) {
767 : // cout << "SDMDataObjectStreamReader::releaseMemory : entering." << endl;
768 79164 : if (sdmDataSubset.actualTimes_ != 0) {
769 : // cout << "actualTimes" << endl;
770 1468 : delete[] sdmDataSubset.actualTimes_;
771 1468 : sdmDataSubset.actualTimes_ = 0;
772 : }
773 79164 : sdmDataSubset.nActualTimes_ = 0;
774 :
775 :
776 79164 : if (sdmDataSubset.actualDurations_ != 0) {
777 : // cout << "actualDurations" << endl;
778 1468 : delete[] sdmDataSubset.actualDurations_;
779 1468 : sdmDataSubset.actualDurations_ = 0;
780 : }
781 79164 : sdmDataSubset.nActualDurations_ = 0;
782 :
783 79164 : if (sdmDataSubset.flags_ != 0) {
784 : // cout << "Flags" << endl;
785 8976 : delete[] sdmDataSubset.flags_;
786 8976 : sdmDataSubset.flags_ = 0;
787 : }
788 79164 : sdmDataSubset.nFlags_ = 0;
789 :
790 79164 : if (sdmDataSubset.zeroLags_ != 0) {
791 : // cout << "zeroLags" << endl;
792 21247 : delete[] sdmDataSubset.zeroLags_;
793 21247 : sdmDataSubset.zeroLags_ = 0;
794 : }
795 79164 : sdmDataSubset.nZeroLags_ = 0;
796 :
797 79164 : if (sdmDataSubset.autoData_ != 0) {
798 : //cout << "autoData_ = " << sdmDataSubset.autoData_ << " before deletion" << endl;
799 40436 : delete[] sdmDataSubset.autoData_;
800 40436 : sdmDataSubset.autoData_ = 0;
801 : }
802 79164 : sdmDataSubset.nAutoData_ = 0;
803 :
804 79164 : if (sdmDataSubset.shortCrossData_ != 0) {
805 : //cout << "shortCrossData_ = " << sdmDataSubset.shortCrossData_ << " before deletion" << endl;
806 22543 : delete[] sdmDataSubset.shortCrossData_;
807 22543 : sdmDataSubset.shortCrossData_ = 0;
808 : }
809 :
810 79164 : if (sdmDataSubset.longCrossData_ != 0) {
811 : // cout << "longCrossData" << endl;
812 2652 : delete[] sdmDataSubset.longCrossData_;
813 2652 : sdmDataSubset.longCrossData_ = 0;
814 : }
815 :
816 79164 : if (sdmDataSubset.floatCrossData_ != 0) {
817 : // cout << "floatCrossData" << endl;
818 1488 : delete[] sdmDataSubset.floatCrossData_;
819 1488 : sdmDataSubset.floatCrossData_ = 0;
820 : }
821 79164 : sdmDataSubset.nCrossData_ = 0;
822 :
823 : // cout << "SDMDataObjectStreamReader::releaseMemory : exiting." << endl;
824 79164 : return;
825 : }
826 : } // end namespace asdmbinaries
827 :
828 : using namespace asdmbinaries;
829 :
830 : #ifdef TEST_CLASS
831 : int main (int argC, char* argV[]) {
832 : if (argC < 2) {
833 : cout << "a.out filename" << endl;
834 : exit(1);
835 : }
836 :
837 : SDMDataObjectStreamReader ssr;
838 : try {
839 : ssr.open(argV[1]);
840 : cout << ssr.toString() << endl;
841 : while (ssr.hasSubset()) {
842 : const SDMDataSubset & sdmdss = ssr.getSubset();
843 : cout << "block of data #" << ssr.currentIntegrationIndex() << " starting at byte #" << ssr.currentIntegrationStartsAt() << endl;
844 : cout << sdmdss.toString(128) << endl;
845 : }
846 : }
847 : catch (SDMDataObjectStreamReaderException& e) {
848 : cout << e.getMessage() << endl;
849 : }
850 : catch (SDMDataObjectParserException& e) {
851 : cout << e.getMessage() << endl;
852 : }
853 : catch (SDMDataObjectException& e) {
854 : cout << e.getMessage() << endl;
855 : }
856 : }
857 : #endif
|
