From 4002b3eb11a25b93a5e5844d930848a539a07531 Mon Sep 17 00:00:00 2001 From: Felipe Elia Date: Mon, 1 Apr 2024 15:12:20 -0300 Subject: [PATCH 1/2] Index csv and txt file contents --- .../classes/Feature/Documents/Documents.php | 2 + tests/cypress/fixtures/csv-file.csv | 130 ++++++++++++++++++ tests/cypress/fixtures/txt-file.txt | 9 ++ .../integration/features/documents.cy.js | 12 +- tests/php/features/TestDocuments.php | 42 ++++++ 5 files changed, 193 insertions(+), 2 deletions(-) create mode 100644 tests/cypress/fixtures/csv-file.csv create mode 100644 tests/cypress/fixtures/txt-file.txt diff --git a/includes/classes/Feature/Documents/Documents.php b/includes/classes/Feature/Documents/Documents.php index 73e2474177..29ee7352ee 100644 --- a/includes/classes/Feature/Documents/Documents.php +++ b/includes/classes/Feature/Documents/Documents.php @@ -399,6 +399,8 @@ public function get_allowed_ingest_mime_types() { 'xlsx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'doc' => 'application/msword', 'docx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'csv' => 'text/csv', + 'txt' => 'text/plain', ) ); } diff --git a/tests/cypress/fixtures/csv-file.csv b/tests/cypress/fixtures/csv-file.csv new file mode 100644 index 0000000000..ee6b058b6d --- /dev/null +++ b/tests/cypress/fixtures/csv-file.csv @@ -0,0 +1,130 @@ +"LatD", "LatM", "LatS", "NS", "LonD", "LonM", "LonS", "EW", "City", "State" + 41, 5, 59, "N", 80, 39, 0, "W", "Youngstown", OH + 42, 52, 48, "N", 97, 23, 23, "W", "Yankton", SD + 46, 35, 59, "N", 120, 30, 36, "W", "Yakima", WA + 42, 16, 12, "N", 71, 48, 0, "W", "Worcester", MA + 43, 37, 48, "N", 89, 46, 11, "W", "Wisconsin Dells", WI + 36, 5, 59, "N", 80, 15, 0, "W", "Winston-Salem", NC + 49, 52, 48, "N", 97, 9, 0, "W", "Winnipeg", MB + 39, 11, 23, "N", 78, 9, 36, "W", "Winchester", VA + 34, 14, 24, "N", 77, 55, 11, "W", "Wilmington", NC + 39, 45, 0, "N", 75, 33, 0, "W", "Wilmington", DE + 48, 9, 0, "N", 103, 37, 12, "W", "Williston", ND + 41, 15, 0, "N", 77, 0, 0, "W", "Williamsport", PA + 37, 40, 48, "N", 82, 16, 47, "W", "Williamson", WV + 33, 54, 0, "N", 98, 29, 23, "W", "Wichita Falls", TX + 37, 41, 23, "N", 97, 20, 23, "W", "Wichita", KS + 40, 4, 11, "N", 80, 43, 12, "W", "Wheeling", WV + 26, 43, 11, "N", 80, 3, 0, "W", "West Palm Beach", FL + 47, 25, 11, "N", 120, 19, 11, "W", "Wenatchee", WA + 41, 25, 11, "N", 122, 23, 23, "W", "Weed", CA + 31, 13, 11, "N", 82, 20, 59, "W", "Waycross", GA + 44, 57, 35, "N", 89, 38, 23, "W", "Wausau", WI + 42, 21, 36, "N", 87, 49, 48, "W", "Waukegan", IL + 44, 54, 0, "N", 97, 6, 36, "W", "Watertown", SD + 43, 58, 47, "N", 75, 55, 11, "W", "Watertown", NY + 42, 30, 0, "N", 92, 20, 23, "W", "Waterloo", IA + 41, 32, 59, "N", 73, 3, 0, "W", "Waterbury", CT + 38, 53, 23, "N", 77, 1, 47, "W", "Washington", DC + 41, 50, 59, "N", 79, 8, 23, "W", "Warren", PA + 46, 4, 11, "N", 118, 19, 48, "W", "Walla Walla", WA + 31, 32, 59, "N", 97, 8, 23, "W", "Waco", TX + 38, 40, 48, "N", 87, 31, 47, "W", "Vincennes", IN + 28, 48, 35, "N", 97, 0, 36, "W", "Victoria", TX + 32, 20, 59, "N", 90, 52, 47, "W", "Vicksburg", MS + 49, 16, 12, "N", 123, 7, 12, "W", "Vancouver", BC + 46, 55, 11, "N", 98, 0, 36, "W", "Valley City", ND + 30, 49, 47, "N", 83, 16, 47, "W", "Valdosta", GA + 43, 6, 36, "N", 75, 13, 48, "W", "Utica", NY + 39, 54, 0, "N", 79, 43, 48, "W", "Uniontown", PA + 32, 20, 59, "N", 95, 18, 0, "W", "Tyler", TX + 42, 33, 36, "N", 114, 28, 12, "W", "Twin Falls", ID + 33, 12, 35, "N", 87, 34, 11, "W", "Tuscaloosa", AL + 34, 15, 35, "N", 88, 42, 35, "W", "Tupelo", MS + 36, 9, 35, "N", 95, 54, 36, "W", "Tulsa", OK + 32, 13, 12, "N", 110, 58, 12, "W", "Tucson", AZ + 37, 10, 11, "N", 104, 30, 36, "W", "Trinidad", CO + 40, 13, 47, "N", 74, 46, 11, "W", "Trenton", NJ + 44, 45, 35, "N", 85, 37, 47, "W", "Traverse City", MI + 43, 39, 0, "N", 79, 22, 47, "W", "Toronto", ON + 39, 2, 59, "N", 95, 40, 11, "W", "Topeka", KS + 41, 39, 0, "N", 83, 32, 24, "W", "Toledo", OH + 33, 25, 48, "N", 94, 3, 0, "W", "Texarkana", TX + 39, 28, 12, "N", 87, 24, 36, "W", "Terre Haute", IN + 27, 57, 0, "N", 82, 26, 59, "W", "Tampa", FL + 30, 27, 0, "N", 84, 16, 47, "W", "Tallahassee", FL + 47, 14, 24, "N", 122, 25, 48, "W", "Tacoma", WA + 43, 2, 59, "N", 76, 9, 0, "W", "Syracuse", NY + 32, 35, 59, "N", 82, 20, 23, "W", "Swainsboro", GA + 33, 55, 11, "N", 80, 20, 59, "W", "Sumter", SC + 40, 59, 24, "N", 75, 11, 24, "W", "Stroudsburg", PA + 37, 57, 35, "N", 121, 17, 24, "W", "Stockton", CA + 44, 31, 12, "N", 89, 34, 11, "W", "Stevens Point", WI + 40, 21, 36, "N", 80, 37, 12, "W", "Steubenville", OH + 40, 37, 11, "N", 103, 13, 12, "W", "Sterling", CO + 38, 9, 0, "N", 79, 4, 11, "W", "Staunton", VA + 39, 55, 11, "N", 83, 48, 35, "W", "Springfield", OH + 37, 13, 12, "N", 93, 17, 24, "W", "Springfield", MO + 42, 5, 59, "N", 72, 35, 23, "W", "Springfield", MA + 39, 47, 59, "N", 89, 39, 0, "W", "Springfield", IL + 47, 40, 11, "N", 117, 24, 36, "W", "Spokane", WA + 41, 40, 48, "N", 86, 15, 0, "W", "South Bend", IN + 43, 32, 24, "N", 96, 43, 48, "W", "Sioux Falls", SD + 42, 29, 24, "N", 96, 23, 23, "W", "Sioux City", IA + 32, 30, 35, "N", 93, 45, 0, "W", "Shreveport", LA + 33, 38, 23, "N", 96, 36, 36, "W", "Sherman", TX + 44, 47, 59, "N", 106, 57, 35, "W", "Sheridan", WY + 35, 13, 47, "N", 96, 40, 48, "W", "Seminole", OK + 32, 25, 11, "N", 87, 1, 11, "W", "Selma", AL + 38, 42, 35, "N", 93, 13, 48, "W", "Sedalia", MO + 47, 35, 59, "N", 122, 19, 48, "W", "Seattle", WA + 41, 24, 35, "N", 75, 40, 11, "W", "Scranton", PA + 41, 52, 11, "N", 103, 39, 36, "W", "Scottsbluff", NB + 42, 49, 11, "N", 73, 56, 59, "W", "Schenectady", NY + 32, 4, 48, "N", 81, 5, 23, "W", "Savannah", GA + 46, 29, 24, "N", 84, 20, 59, "W", "Sault Sainte Marie", MI + 27, 20, 24, "N", 82, 31, 47, "W", "Sarasota", FL + 38, 26, 23, "N", 122, 43, 12, "W", "Santa Rosa", CA + 35, 40, 48, "N", 105, 56, 59, "W", "Santa Fe", NM + 34, 25, 11, "N", 119, 41, 59, "W", "Santa Barbara", CA + 33, 45, 35, "N", 117, 52, 12, "W", "Santa Ana", CA + 37, 20, 24, "N", 121, 52, 47, "W", "San Jose", CA + 37, 46, 47, "N", 122, 25, 11, "W", "San Francisco", CA + 41, 27, 0, "N", 82, 42, 35, "W", "Sandusky", OH + 32, 42, 35, "N", 117, 9, 0, "W", "San Diego", CA + 34, 6, 36, "N", 117, 18, 35, "W", "San Bernardino", CA + 29, 25, 12, "N", 98, 30, 0, "W", "San Antonio", TX + 31, 27, 35, "N", 100, 26, 24, "W", "San Angelo", TX + 40, 45, 35, "N", 111, 52, 47, "W", "Salt Lake City", UT + 38, 22, 11, "N", 75, 35, 59, "W", "Salisbury", MD + 36, 40, 11, "N", 121, 39, 0, "W", "Salinas", CA + 38, 50, 24, "N", 97, 36, 36, "W", "Salina", KS + 38, 31, 47, "N", 106, 0, 0, "W", "Salida", CO + 44, 56, 23, "N", 123, 1, 47, "W", "Salem", OR + 44, 57, 0, "N", 93, 5, 59, "W", "Saint Paul", MN + 38, 37, 11, "N", 90, 11, 24, "W", "Saint Louis", MO + 39, 46, 12, "N", 94, 50, 23, "W", "Saint Joseph", MO + 42, 5, 59, "N", 86, 28, 48, "W", "Saint Joseph", MI + 44, 25, 11, "N", 72, 1, 11, "W", "Saint Johnsbury", VT + 45, 34, 11, "N", 94, 10, 11, "W", "Saint Cloud", MN + 29, 53, 23, "N", 81, 19, 11, "W", "Saint Augustine", FL + 43, 25, 48, "N", 83, 56, 24, "W", "Saginaw", MI + 38, 35, 24, "N", 121, 29, 23, "W", "Sacramento", CA + 43, 36, 36, "N", 72, 58, 12, "W", "Rutland", VT + 33, 24, 0, "N", 104, 31, 47, "W", "Roswell", NM + 35, 56, 23, "N", 77, 48, 0, "W", "Rocky Mount", NC + 41, 35, 24, "N", 109, 13, 48, "W", "Rock Springs", WY + 42, 16, 12, "N", 89, 5, 59, "W", "Rockford", IL + 43, 9, 35, "N", 77, 36, 36, "W", "Rochester", NY + 44, 1, 12, "N", 92, 27, 35, "W", "Rochester", MN + 37, 16, 12, "N", 79, 56, 24, "W", "Roanoke", VA + 37, 32, 24, "N", 77, 26, 59, "W", "Richmond", VA + 39, 49, 48, "N", 84, 53, 23, "W", "Richmond", IN + 38, 46, 12, "N", 112, 5, 23, "W", "Richfield", UT + 45, 38, 23, "N", 89, 25, 11, "W", "Rhinelander", WI + 39, 31, 12, "N", 119, 48, 35, "W", "Reno", NV + 50, 25, 11, "N", 104, 39, 0, "W", "Regina", SA + 40, 10, 48, "N", 122, 14, 23, "W", "Red Bluff", CA + 40, 19, 48, "N", 75, 55, 48, "W", "Reading", PA + 41, 9, 35, "N", 81, 14, 23, "W", "Ravenna", OH + diff --git a/tests/cypress/fixtures/txt-file.txt b/tests/cypress/fixtures/txt-file.txt new file mode 100644 index 0000000000..9cd0d6d3e7 --- /dev/null +++ b/tests/cypress/fixtures/txt-file.txt @@ -0,0 +1,9 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque quis mi at nisl facilisis ultrices. Vivamus efficitur a dolor sed varius. Aenean turpis ipsum, rhoncus sed mi in, tincidunt pharetra ante. Mauris egestas tortor vel metus elementum mollis. In nec pretium velit. Aliquam nibh nisi, fermentum a aliquet nec, ullamcorper ac nisi. Sed et faucibus eros. Fusce est lectus, volutpat in massa eu, venenatis tristique mauris. Cras bibendum, nibh at iaculis vestibulum, lorem ipsum porta justo, eu commodo neque orci a turpis. Sed et erat urna. Fusce euismod lacus lorem, quis suscipit sem tempus et. Curabitur interdum id turpis ac viverra. + +Praesent nec mollis dui. Pellentesque sit amet vehicula orci. Proin eros neque, commodo scelerisque nisi vitae, rhoncus pretium enim. Integer convallis tortor sem, nec maximus urna tempus id. Curabitur et neque scelerisque, faucibus elit mattis, aliquam tellus. Quisque sit amet hendrerit lacus, nec semper mi. Fusce pretium nisi at orci vestibulum, non interdum magna malesuada. Cras vestibulum vitae tellus in faucibus. Donec tempor hendrerit dignissim. + +Nam nec viverra justo. Quisque convallis dictum dolor, vel pellentesque lectus pharetra a. Nunc purus massa, ultrices sit amet nulla vel, pharetra lacinia tortor. Etiam ac risus non magna fringilla feugiat faucibus sit amet quam. Aliquam erat volutpat. Donec ultricies orci tortor, ac aliquam orci tempor eget. Donec risus dui, suscipit nec ante ac, auctor varius lectus. Ut id lorem ut leo tincidunt dictum. Praesent sapien justo, luctus non dolor vel, sollicitudin imperdiet elit. Etiam eros metus, ornare nec finibus non, consequat id nisl. Morbi molestie in turpis at suscipit. Nam risus felis, fringilla vitae purus nec, ornare fringilla metus. + +Sed euismod nisl purus, vel consectetur purus consectetur sed. Suspendisse eget lacus fermentum, convallis lectus eget, laoreet mauris. Morbi id varius neque. In hac habitasse platea dictumst. Duis ut eros aliquet, aliquam enim non, euismod sem. Duis suscipit odio dolor, sit amet viverra orci cursus feugiat. Phasellus scelerisque, justo at auctor eleifend, diam purus fringilla dolor, at malesuada augue risus tristique tortor. Vivamus ac commodo augue. Aliquam sapien mi, interdum ut ipsum nec, sagittis semper lacus. Maecenas magna odio, fringilla tempus purus quis, auctor rutrum diam. Proin blandit velit quis diam convallis facilisis. + +Aenean lobortis elit ac posuere imperdiet. Maecenas interdum neque sapien, sollicitudin interdum eros elementum volutpat. Nunc at lorem neque. Pellentesque lacinia hendrerit sapien non commodo. Phasellus sollicitudin diam ac sem ultrices vestibulum. Cras sollicitudin dui id lectus pellentesque lacinia. In eu odio sem. Donec ac lobortis eros, vehicula rutrum orci. Donec tristique massa a iaculis dapibus. diff --git a/tests/cypress/integration/features/documents.cy.js b/tests/cypress/integration/features/documents.cy.js index 26f7627b7e..c567c9a1fa 100644 --- a/tests/cypress/integration/features/documents.cy.js +++ b/tests/cypress/integration/features/documents.cy.js @@ -69,16 +69,24 @@ describe('Documents Feature', () => { }); }); - it('Can search .pptx', () => { + it('Can search .pptx, .txt, and .csv files', () => { cy.login(); enableDocumentsFeature(); + uploadFile( 'pptx-file.pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation', ); + uploadFile('txt-file.txt', 'text/plain'); + uploadFile('csv-file.csv', 'text/csv'); cy.visit('/?s=dummy+slide'); - cy.get('.hentry').should('contain.text', 'pptx-file'); + + cy.visit('/?s=Curabitur+interdum+id+turpis+ac+viverra'); + cy.get('.hentry').should('contain.text', 'txt-file'); + + cy.visit('/?s=Winchester'); + cy.get('.hentry').should('contain.text', 'csv-file'); }); }); diff --git a/tests/php/features/TestDocuments.php b/tests/php/features/TestDocuments.php index 437ba0f6fd..ee9b850363 100644 --- a/tests/php/features/TestDocuments.php +++ b/tests/php/features/TestDocuments.php @@ -258,4 +258,46 @@ public function testQueryForAttachments() { $this->assertTrue( $query->elasticsearch_success ); $this->assertEquals( 1, $query->post_count ); } + + /** + * Test the `get_allowed_ingest_mime_types` method. + * + * @since 5.1.0 + * @group documents + */ + public function test_get_allowed_ingest_mime_types() { + $feature = ElasticPress\Features::factory()->get_registered_feature( 'documents' ); + + $expected = [ + 'pdf' => 'application/pdf', + 'ppt' => 'application/vnd.ms-powerpoint', + 'pptx' => 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + 'xls' => 'application/vnd.ms-excel', + 'xlsx' => 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + 'doc' => 'application/msword', + 'docx' => 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + 'csv' => 'text/csv', + 'txt' => 'text/plain', + ]; + $this->assertSame( $expected, $feature->get_allowed_ingest_mime_types() ); + } + + + /** + * Test the `ep_allowed_documents_ingest_mime_types` filter. + * + * @since 5.1.0 + * @group documents + */ + public function test_ep_allowed_documents_ingest_mime_types_filter() { + $feature = ElasticPress\Features::factory()->get_registered_feature( 'documents' ); + + $change_filter = function ( $allowed_mime_types ) { + $allowed_mime_types['test'] = 'text/test'; + return $allowed_mime_types; + }; + add_filter( 'ep_allowed_documents_ingest_mime_types', $change_filter ); + + $this->assertSame( 'text/test', $feature->get_allowed_ingest_mime_types()['test'] ); + } } From e24f609158dfa1cee6d1916d3e487f02b9d098af Mon Sep 17 00:00:00 2001 From: Felipe Elia Date: Mon, 1 Apr 2024 15:54:52 -0300 Subject: [PATCH 2/2] Allow txt and csv uploads --- .../integration/features/documents.cy.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/cypress/integration/features/documents.cy.js b/tests/cypress/integration/features/documents.cy.js index c567c9a1fa..b8f978f000 100644 --- a/tests/cypress/integration/features/documents.cy.js +++ b/tests/cypress/integration/features/documents.cy.js @@ -77,16 +77,21 @@ describe('Documents Feature', () => { 'pptx-file.pptx', 'application/vnd.openxmlformats-officedocument.presentationml.presentation', ); - uploadFile('txt-file.txt', 'text/plain'); - uploadFile('csv-file.csv', 'text/csv'); cy.visit('/?s=dummy+slide'); cy.get('.hentry').should('contain.text', 'pptx-file'); - cy.visit('/?s=Curabitur+interdum+id+turpis+ac+viverra'); - cy.get('.hentry').should('contain.text', 'txt-file'); + cy.wpCli('config set ALLOW_UNFILTERED_UPLOADS true --raw').then(() => { + uploadFile('txt-file.txt', 'text/plain'); + uploadFile('csv-file.csv', 'text/csv'); - cy.visit('/?s=Winchester'); - cy.get('.hentry').should('contain.text', 'csv-file'); + cy.visit('/?s=Curabitur+interdum+id+turpis+ac+viverra'); + cy.get('.hentry').should('contain.text', 'txt-file'); + + cy.visit('/?s=Winchester'); + cy.get('.hentry').should('contain.text', 'csv-file'); + + cy.wpCli('config set ALLOW_UNFILTERED_UPLOADS false --raw'); + }); }); });