[Libreoffice-commits] .: 2 commits - sc/inc sc/qa
Eike Rathke
erack at kemper.freedesktop.org
Sat Apr 14 11:25:26 PDT 2012
sc/inc/orcus/csv_parser.hpp | 6
sc/qa/unit/data/contentCSV/fdo48621_broken_quotes_exported.csv | 90 ++++++++++
sc/qa/unit/data/csv/fdo48621_broken_quotes.csv | 88 +++++++++
sc/qa/unit/subsequent_filters-test.cxx | 33 +++
4 files changed, 213 insertions(+), 4 deletions(-)
New commits:
commit d3b74479a7d0f6f02c86e72be14ae3880726ce59
Author: Eike Rathke <erack at redhat.com>
Date: Sat Apr 14 20:22:56 2012 +0200
fix orcus to not fiddle with literal strings
diff --git a/sc/inc/orcus/csv_parser.hpp b/sc/inc/orcus/csv_parser.hpp
index 3e52bc1..77f8a3c 100644
--- a/sc/inc/orcus/csv_parser.hpp
+++ b/sc/inc/orcus/csv_parser.hpp
@@ -256,14 +256,14 @@ void csv_parser<_Handler>::quoted_cell()
}
// Closing quote.
- push_cell_value(p0, len-1);
+ m_handler.cell(p0, len-1);
next();
skip_blanks();
return;
}
// Stream ended prematurely. Handle it gracefully.
- push_cell_value(p0, len);
+ m_handler.cell(p0, len);
next();
skip_blanks();
}
@@ -309,7 +309,7 @@ void csv_parser<_Handler>::parse_cell_with_quote(const char* p0, size_t len0)
// buffer, push the value to the handler, and exit normally.
append_to_cell_buf(p_cur, cur_len);
- push_cell_value(&m_cell_buf[0], m_cell_buf_size);
+ m_handler.cell(&m_cell_buf[0], m_cell_buf_size);
next();
skip_blanks();
return;
commit 1e94b35b3d8d25636a5dc6ad99f81399617ad105
Author: Eike Rathke <erack at redhat.com>
Date: Sat Apr 14 20:22:55 2012 +0200
unit test for CSV import fixes related to fdo#48621
diff --git a/sc/qa/unit/data/contentCSV/fdo48621_broken_quotes_exported.csv b/sc/qa/unit/data/contentCSV/fdo48621_broken_quotes_exported.csv
new file mode 100644
index 0000000..b1c95af
--- /dev/null
+++ b/sc/qa/unit/data/contentCSV/fdo48621_broken_quotes_exported.csv
@@ -0,0 +1,90 @@
+i80358,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"abc"" d ""ef",shall be:,"abc"" d ""ef",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"a""b, ""a",shall be:,"a""b, ""a",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i78926_input.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+2,5,GR010 015 1001,/image/grimas/015/1001.jpg,/image/grimas/015/1001.jpg,/image/grimas/015/1001.jpg,,,,,,,,,,,,,5.45,0,0000-00-00 00:00:00,0000-00-00 00:00:00,0000-00-00 00:00:00,0.05,1,2,81,4,0,1,0,2000,0,0,0,"0
+"
+3,5,GR010 015 1002,/image/grimas/015/1002.jpg,/image/grimas/015/1002.jpg,/image/grimas/015/1002.jpg,,,,,,,,,,,,,5.45,0,0000-00-00 00:00:00,0000-00-00 00:00:00,0000-00-00 00:00:00,0.05,1,2,81,4,0,0,0,2000,0,0,0,"0
+"
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i78926_input.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+772,3,SPAZZOLA ARIA BABYLISS 32,image/APP.ELET/Varie/Babyliss-Spazzola-Elettrica.jpg,image/APP.ELET/Varie/Babyliss-Spazzola-Elettrica.jpg,image/APP.ELET/Varie/Babyliss-Spazzola-Elettrica.jpg,,,,,,,,,,,,,33,0,2006-11-07 09:26:16,2007-05-24 17:54:24,NULL,1,1,2,86,150,0,0,0,2000,0,,,
+773,1,"BIGODINI RISCALDANTI \""BA",image/APP.ELET/Varie/Bigodini-caldi.jpg,image/APP.ELET/Varie/Bigodini-caldi.jpg,image/APP.ELET/Varie/Bigodini-caldi.jpg,,,,,,,,,,,,,57.91,0,2006-11-07 09:26:16,2006-11-07 09:26:16,0000-00-00 00:00:00,0.7,1,2,86,152,0,1,0,2000,0,0,0,0
+774,3,"PIASTRA \""BABYLISS\"" PRO C",image/APP.ELET/piastre/Piastre-Babyliss-2564E.jpg,image/APP.ELET/piastre/Piastre-Babyliss-2564E.jpg,image/APP.ELET/piastre/Piastre-Babyliss-2564E.jpg,,,,,,,,,,,,,78,0,2006-11-07 09:26:16,2006-12-13 19:18:31,0000-00-00 00:00:00,0.7,1,2,86,148,0,0,0,2000,0,0,0,0
+775,0,PHON ULTRON MAXI 2200W+1,image/APP.ELET/phon/Phon-Ultron-1500W.jpg,image/APP.ELET/phon/Phon-Ultron-1500W.jpg,image/APP.ELET/phon/Phon-Ultron-1500W.jpg,,,,,,,,,,,,,26.66,0,2006-11-07 09:26:16,2006-12-13 18:57:03,0000-00-00 00:00:00,0.7,1,2,101,158,0,0,0,2000,0,0,0,0
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i78926_inconsistent_double_quote_behavior.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Description,Description,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+The following text breaks everything,"OpenOffice.org calc doesnt "" like quotes embedded in fields.",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+This field will fix everything,"Here is the "" fix",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+First fixed record,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+This field will break things again,"Mismatched quote "" must be after some quoted text",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+This field will fix everything,"Here is the "" fix",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+First fixed record,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+This next field will /not/ break things because /this/ field is not quoted,"Mismatched quote "" must be after some quoted text",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+Here is some data,Here is some data,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i80385_test1.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+test,"a""b, ""a"," d""a""c ", m ,j ,d,"b""A""","D""E","f,1","a,b","de""b,a",,,,,,,,,,,,,,,,,,,,,,,,,
+"abcdef"" test ""abc","def""g""h","def""gh""",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"this is
+a test","yes
+it
+works",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+No it doesn't,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"yes it,"",""does",no,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"""""a,cd""e
+""a""b""
+",<- needed to end test file here,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i80385_test2.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+test,"a""b, ""a"," d""a""c ", m ,j ,d,"b""A""","D""E","f,1","a,b","de""b,a
+""abcdef"" test ""abc","def""g""h","def""gh""",,,,,,,,,,,,,,,,,,,,,,,
+"this is
+a test","yes
+it
+works",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+No it doesn't,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"yes it,"",""does",no,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"""""a,cd""e
+""a""b""
+",<- needed to end test file here,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i80385_test3.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+test,"a""b, ""a"," d""a""c ", m ,j ,d,"b""A""","D""E","f,1","a,b","de""b,a",,,,,,,,,,,,,,,,,,,,,,,,,
+"abcdef"" test ""abc","def""g""h","def""gh""",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"this is
+a test","yes
+it
+works",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+No it doesn't,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"yes it,"",""does",no,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"""""a,cd""e
+""a""b""",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+i80385_test4.csv,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+test,"a""b, ""a"," d""a""c ", m ,j ,d,"b""A""","D""E","f,1","a,b","de""b,a
+""abcdef"" test ""abc","def""g""h","def""gh""",,,,,,,,,,,,,,,,,,,,,,,
+"this is
+a test","yes
+it
+works",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+No it doesn't,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"yes it,"",""does",no,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+"""""a,cd""e
+""a""b""",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
diff --git a/sc/qa/unit/data/csv/fdo48621_broken_quotes.csv b/sc/qa/unit/data/csv/fdo48621_broken_quotes.csv
new file mode 100644
index 0000000..dbefc24
--- /dev/null
+++ b/sc/qa/unit/data/csv/fdo48621_broken_quotes.csv
@@ -0,0 +1,88 @@
+i80358
+"abc" d "ef",shall be:,"abc"" d ""ef"
+"a"b, "a",shall be:,"a""b, ""a"
+
+i78926_input.csv
+"2","5","GR010 015 1001","/image/grimas/015/1001.jpg","/image/grimas/015/1001.jpg","/image/grimas/015/1001.jpg",,,,,,,,,,,,,"5.45","0.00","0000-00-00 00:00:00","0000-00-00 00:00:00","0000-00-00 00:00:00","0.05","1","2","81","4","0","1","0","2000","0","0","0","0
"
+"3","5","GR010 015 1002","/image/grimas/015/1002.jpg","/image/grimas/015/1002.jpg","/image/grimas/015/1002.jpg",,,,,,,,,,,,,"5.45","0.00","0000-00-00 00:00:00","0000-00-00 00:00:00","0000-00-00 00:00:00","0.05","1","2","81","4","0","0","0","2000","0","0","0","0
"
+
+i78926_input.csv
+"772","3","SPAZZOLA ARIA BABYLISS 32","image/APP.ELET/Varie/Babyliss-Spazzola-Elettrica.jpg","image/APP.ELET/Varie/Babyliss-Spazzola-Elettrica.jpg","image/APP.ELET/Varie/Babyliss-Spazzola-Elettrica.jpg",,,,,,,,,,,,,"33.00","0.00","2006-11-07 09:26:16","2007-05-24 17:54:24",NULL,"1.00","1","2","86","150","0","0","0","2000","0",,,
+"773","1","BIGODINI RISCALDANTI \"BA","image/APP.ELET/Varie/Bigodini-caldi.jpg","image/APP.ELET/Varie/Bigodini-caldi.jpg","image/APP.ELET/Varie/Bigodini-caldi.jpg",,,,,,,,,,,,,"57.91","0.00","2006-11-07 09:26:16","2006-11-07 09:26:16","0000-00-00 00:00:00","0.70","1","2","86","152","0","1","0","2000","0","0","0","0"
+"774","3","PIASTRA \"BABYLISS\" PRO C","image/APP.ELET/piastre/Piastre-Babyliss-2564E.jpg","image/APP.ELET/piastre/Piastre-Babyliss-2564E.jpg","image/APP.ELET/piastre/Piastre-Babyliss-2564E.jpg",,,,,,,,,,,,,"78.00","0.00","2006-11-07 09:26:16","2006-12-13 19:18:31","0000-00-00 00:00:00","0.70","1","2","86","148","0","0","0","2000","0","0","0","0"
+"775","0","PHON ULTRON MAXI 2200W+1","image/APP.ELET/phon/Phon-Ultron-1500W.jpg","image/APP.ELET/phon/Phon-Ultron-1500W.jpg","image/APP.ELET/phon/Phon-Ultron-1500W.jpg",,,,,,,,,,,,,"26.66","0.00","2006-11-07 09:26:16","2006-12-13 18:57:03","0000-00-00 00:00:00","0.70","1","2","101","158","0","0","0","2000","0","0","0","0"
+
+i78926_inconsistent_double_quote_behavior.csv
+Description,Description
+"The following text breaks everything",OpenOffice.org calc doesnt " like quotes embedded in fields.
+Here is some data,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+This field will fix everything,Here is the " fix
+First fixed record,Here is some data
+Here is some data,Here is some data
+"This field will break things again",Mismatched quote " must be after some quoted text
+Here is some data,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+This field will fix everything,Here is the " fix
+First fixed record,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+This next field will /not/ break things because /this/ field is not quoted,Mismatched quote " must be after some quoted text
+Here is some data,Here is some data
+Here is some data,Here is some data
+Here is some data,Here is some data
+
+i80385_test1.csv
+test,"a"b, "a", d"a"c , m , "j" ,d,b"A",D"E,"f,1","a,b","de"b,a"
+"abcdef" test "abc","def"g"h",def"gh"
+"this is
+a test","yes
+it
+works",
+No it doesn't
+"yes it,"","does","no"
+""""a,cd""e
+"a""b""
+",<- needed to end test file here
+
+i80385_test2.csv
+test,"a"b, "a", d"a"c , m , "j" ,d,b"A",D"E,"f,1","a,b","de"b,a
+"abcdef" test "abc","def"g"h",def"gh"
+"this is
+a test","yes
+it
+works",
+No it doesn't
+"yes it,"","does","no"
+""""a,cd""e
+"a""b""
+",<- needed to end test file here
+
+i80385_test3.csv
+test,"a"b, "a", d"a"c , m , "j" ,d,b"A",D"E,"f,1","a,b","de"b,a"
+"abcdef" test "abc","def"g"h",def"gh"
+"this is
+a test","yes
+it
+works",
+No it doesn't
+"yes it,"","does","no"
+""""a,cd""e
+"a""b"""
+
+i80385_test4.csv
+test,"a"b, "a", d"a"c , m , "j" ,d,b"A",D"E,"f,1","a,b","de"b,a
+"abcdef" test "abc","def"g"h",def"gh"
+"this is
+a test","yes
+it
+works",
+No it doesn't
+"yes it,"","does","no"
+""""a,cd""e
+"a""b"""
diff --git a/sc/qa/unit/subsequent_filters-test.cxx b/sc/qa/unit/subsequent_filters-test.cxx
index f2ed04b..f7d5d2b 100644
--- a/sc/qa/unit/subsequent_filters-test.cxx
+++ b/sc/qa/unit/subsequent_filters-test.cxx
@@ -52,10 +52,12 @@
#define ODS_FORMAT_TYPE 50331943
#define XLS_FORMAT_TYPE 318767171
#define XLSX_FORMAT_TYPE 268959811
+#define CSV_FORMAT_TYPE 195
#define ODS 0
#define XLS 1
#define XLSX 2
+#define CSV 3
using namespace ::com::sun::star;
using namespace ::com::sun::star::uno;
@@ -69,7 +71,8 @@ struct FileFormat {
FileFormat aFileFormats[] = {
{ "ods" , "calc8", "", ODS_FORMAT_TYPE },
{ "xls" , "MS Excel 97", "calc_MS_EXCEL_97", XLS_FORMAT_TYPE },
- { "xlsx", "Calc MS Excel 2007 XML" , "MS Excel 2007 XML", XLSX_FORMAT_TYPE }
+ { "xlsx", "Calc MS Excel 2007 XML" , "MS Excel 2007 XML", XLSX_FORMAT_TYPE },
+ { "csv" , "Text - txt - csv (StarCalc)", "calc_Text_txt_csv_StarCalc", CSV_FORMAT_TYPE }
};
}
@@ -113,6 +116,7 @@ public:
void testBugFixesODS();
void testBugFixesXLS();
void testBugFixesXLSX();
+ void testBrokenQuotesCSV();
//misc tests unrelated to the import filters
void testPasswordNew();
@@ -135,6 +139,9 @@ public:
CPPUNIT_TEST(testBugFixesODS);
CPPUNIT_TEST(testBugFixesXLS);
CPPUNIT_TEST(testBugFixesXLSX);
+#if 0
+ CPPUNIT_TEST(testBrokenQuotesCSV);
+#endif
//disable testPassword on MacOSX due to problems with libsqlite3
//also crashes on DragonFly due to problems with nss/nspr headers
#if !defined(MACOSX) && !defined(DRAGONFLY) && !defined(WNT)
@@ -636,6 +643,30 @@ void ScFiltersTest::testBugFixesXLSX()
xDocSh->DoClose();
}
+void ScFiltersTest::testBrokenQuotesCSV()
+{
+ const rtl::OUString aFileNameBase(RTL_CONSTASCII_USTRINGPARAM("fdo48621_broken_quotes."));
+ rtl::OUString aFileExtension(aFileFormats[CSV].pName, strlen(aFileFormats[CSV].pName), RTL_TEXTENCODING_UTF8 );
+ rtl::OUString aFilterName(aFileFormats[CSV].pFilterName, strlen(aFileFormats[CSV].pFilterName), RTL_TEXTENCODING_UTF8) ;
+ rtl::OUString aFileName;
+ createFileURL(aFileNameBase, aFileExtension, aFileName);
+ rtl::OUString aFilterType(aFileFormats[CSV].pTypeName, strlen(aFileFormats[CSV].pTypeName), RTL_TEXTENCODING_UTF8);
+ std::cout << aFileFormats[CSV].pName << " Test" << std::endl;
+ ScDocShellRef xDocSh = load (aFilterName, aFileName, rtl::OUString(), aFilterType, aFileFormats[CSV].nFormatType);
+
+ CPPUNIT_ASSERT_MESSAGE("Failed to load fdo48621_broken_quotes.csv", xDocSh.Is());
+ ScDocument* pDoc = xDocSh->GetDocument();
+ CPPUNIT_ASSERT_MESSAGE("No Document", pDoc); //remove with first test
+
+ rtl::OUString aSheet2CSV(RTL_CONSTASCII_USTRINGPARAM("fdo48621_broken_quotes_exported."));
+ rtl::OUString aCSVPath;
+ createCSVPath( aSheet2CSV, aCSVPath );
+ // fdo#48621
+ testFile( aCSVPath, pDoc, 0, PureString);
+
+ xDocSh->DoClose();
+}
+
void ScFiltersTest::testPassword_Impl(const rtl::OUString& aFileNameBase)
{
rtl::OUString aFileExtension(aFileFormats[0].pName, strlen(aFileFormats[0].pName), RTL_TEXTENCODING_UTF8 );
More information about the Libreoffice-commits
mailing list