Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improved remove duplicates #1034

Merged
merged 15 commits into from
Dec 28, 2024
+106 −5
Merged

Improved remove duplicates #1034

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
f77e293
Improve remove duplicates
girishpanchal30 Dec 17, 2024
ae42195
fix: update upsell link
girishpanchal30 Dec 17, 2024
c4e1325
Add multiple tags support
girishpanchal30 Dec 18, 2024
eb17570
Encode the tag value if the value type is an object
girishpanchal30 Dec 18, 2024
c29d232
Create unique meta key
girishpanchal30 Dec 18, 2024
564ef69
e2e: check upsell link
girishpanchal30 Dec 19, 2024
5cf176c
Add phpunit tests
girishpanchal30 Dec 19, 2024
4f712c7
Improve PHPUnit testcases
girishpanchal30 Dec 19, 2024
0a57dc4
fix: change meta key
girishpanchal30 Dec 20, 2024
a572991
fix: import count
girishpanchal30 Dec 20, 2024
cd9704e
fix: php warning
girishpanchal30 Dec 20, 2024
97f050a
fix: e2e and phpunit testcase error
girishpanchal30 Dec 27, 2024
7b6020d
fix: phpunit testcase error
girishpanchal30 Dec 27, 2024
94e313c
Merge branch 'development' into feat/1006
HardeepAsrani Dec 28, 2024
d5a33ba
Merge branch 'development' into feat/1006
HardeepAsrani Dec 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 38 additions & 2 deletions includes/admin/feedzy-rss-feeds-import.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,7 @@ public function feedzy_import_feed_options() {
$import_auto_translation = get_post_meta( $post->ID, 'import_auto_translation', true );
$import_auto_translation = 'yes' === $import_auto_translation ? 'checked' : '';
$import_translation_lang = get_post_meta( $post->ID, 'import_auto_translation_lang', true );
$mark_duplicate_tag = get_post_meta( $post->ID, 'mark_duplicate_tag', true );
$import_post_author = get_post_meta( $post->ID, 'import_post_author', true );
$filter_conditions = get_post_meta( $post->ID, 'filter_conditions', true );

Expand Down Expand Up @@ -1028,6 +1029,9 @@ private function get_complete_import_status( $post_id ) {
if ( $items ) {
$status['cumulative'] = count( $items );
}
if ( ! empty( $status['duplicates'] ) ) {
$status['total'] = absint( $status['total'] - count( $status['duplicates'] ) );
}

return $status;
}
Expand Down Expand Up @@ -1406,6 +1410,8 @@ private function run_job( $job, $max ) {
$import_translation_lang = get_post_meta( $job->ID, 'import_auto_translation_lang', true );
$filter_conditions = get_post_meta( $job->ID, 'filter_conditions', true );
$import_post_author = get_post_meta( $job->ID, 'import_post_author', true );
$mark_duplicate_tag = get_post_meta( $job->ID, 'mark_duplicate_tag', true );
$mark_duplicate_tag = feedzy_is_pro() && ! empty( $mark_duplicate_tag ) ? preg_replace( '/[\[\]#]/', '', $mark_duplicate_tag ) : '';
$max = $import_feed_limit;

if ( empty( $filter_conditions ) ) {
Expand Down Expand Up @@ -1561,11 +1567,36 @@ private function run_job( $job, $max ) {
$is_duplicate = $use_new_hash ? in_array( $item_hash, $imported_items_new, true ) : in_array( $item_hash, $imported_items_old, true );
$items_found[ $item['item_url'] ] = $item['item_title'];

$duplicate_tag_value = array();
$mark_duplicate_key = 'item_url';
if ( 'yes' === $import_remove_duplicates && ! $is_duplicate ) {
$is_duplicate_post = $this->is_duplicate_post( $import_post_type, 'feedzy_item_url', esc_url_raw( $item['item_url'] ) );
if ( ! empty( $mark_duplicate_tag ) ) {
$mark_duplicate_tag = is_string( $mark_duplicate_tag ) ? explode( ',', $mark_duplicate_tag ) : $mark_duplicate_tag;
$mark_duplicate_tag = array_map( 'trim', $mark_duplicate_tag );
$duplicate_tag_value = array_map(
function ( $tag ) use ( $item_obj, $item ) {
if ( str_contains( $tag, 'item_custom' ) && $this->feedzy_is_business() ) {
$tag = apply_filters( 'feedzy_parse_custom_tags', "[#$tag]", $item_obj );
} elseif ( isset( $item[ $tag ] ) ) {
$tag = isset( $item[ $tag ] ) ? is_object( $item[ $tag ] ) ? wp_json_encode( $item[ $tag ] ) : $item[ $tag ] : '';
}
return $tag;
},
$mark_duplicate_tag
);
}
if ( ! empty( $duplicate_tag_value ) ) {
$duplicate_tag_value = implode( ' ', $duplicate_tag_value );
$duplicate_tag_value = substr( sanitize_key( wp_strip_all_tags( $duplicate_tag_value ) ), 0, apply_filters( 'feedzy_mark_duplicate_content_limit', 256 ) );
$mark_duplicate_key = 'mark_duplicate';
} else {
$duplicate_tag_value = esc_url_raw( $item['item_url'] );
}
$is_duplicate_post = $this->is_duplicate_post( $import_post_type, 'feedzy_' . $mark_duplicate_key, $duplicate_tag_value );
if ( ! empty( $is_duplicate_post ) ) {
foreach ( $is_duplicate_post as $p ) {
$found_duplicates[] = get_post_meta( $p, 'feedzy_item_url', true );
$found_duplicates[ $item_hash ] = get_post_meta( $p, 'feedzy_' . $mark_duplicate_key, true );
$duplicates[ $item['item_url'] ] = $item['item_title'];
wp_delete_post( $p, true );
}
}
Expand Down Expand Up @@ -2108,6 +2139,11 @@ function( $term ) {
update_post_meta( $new_post_id, 'feedzy_job', $job->ID );
update_post_meta( $new_post_id, 'feedzy_item_author', sanitize_text_field( $author ) );

// Verify that the `$mark_duplicate_key` does not match `'item_url'` to ensure the condition applies only when a different tag is specified.
if ( $mark_duplicate_key && 'item_url' !== $mark_duplicate_key ) {
update_post_meta( $new_post_id, 'feedzy_' . $mark_duplicate_key, $duplicate_tag_value );
}

// we can use this to associate the items that were imported in a particular run.
update_post_meta( $new_post_id, 'feedzy_job_time', $last_run );

Expand Down
19 changes: 17 additions & 2 deletions includes/views/import-metabox-edit.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -657,8 +657,23 @@ class="fz-switch-toggle" type="checkbox" value="yes"
</div>
</div>

<div class="form-block form-block-two-column <?php echo feedzy_is_legacyv5() ? '' : esc_attr( apply_filters( 'feedzy_upsell_class', '' ) ); ?>">
<?php echo feedzy_is_legacyv5() ? '' : wp_kses_post( apply_filters( 'feedzy_upsell_content', '', 'count', 'import' ) ); ?>
<div class="form-block form-block-two-column <?php echo esc_attr( apply_filters( 'feedzy_upsell_class', '' ) ); ?>">
<?php echo wp_kses_post( apply_filters( 'feedzy_upsell_content', '', 'remove-duplicates', 'import' ) ); ?>
<div class="fz-left">
<h4 class="h4"><?php esc_html_e( 'Mark Duplicates By', 'feedzy-rss-feeds' ); ?> <?php echo ! feedzy_is_pro() ? ' <span class="pro-label">PRO</span>' : ''; ?></h4>
</div>
<div class="fz-right">
<div class="fz-form-group">
<label class="form-label"><?php esc_html_e( 'Mark the duplicate items using the magic tag value', 'feedzy-rss-feeds' ); ?></label>
<input type="text" id="feedzy_mark_duplicate" name="feedzy_meta_data[mark_duplicate_tag]" class="form-control" value="<?php echo esc_attr( $mark_duplicate_tag ); ?>"<?php disabled( true, 'checked' !== $import_remove_duplicates ); ?> />
<div class="help-text pt-8">
<?php esc_html_e( 'Helpful if you want to identify and remove duplicate items. Enter a magic tag to mark duplicates based on its value, e.g: [#item_title], [#item_content], [#item_url] etc.', 'feedzy-rss-feeds' ); ?>
</div>
</div>
</div>
</div>

<div class="form-block form-block-two-column">
<div class="fz-left">
<h4 class="h4"><?php esc_html_e( 'Items Count', 'feedzy-rss-feeds' ); ?><?php echo ! feedzy_is_pro() && ! feedzy_is_legacyv5() ? ' <span class="pro-label">PRO</span>' : ''; ?></h4>
</div>
Expand Down
8 changes: 8 additions & 0 deletions includes/views/js/import-metabox-edit.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -661,6 +661,14 @@
}
} );

$( document ).on( 'change', 'input#remove-duplicates', function() {
if ( $(this).is(':checked') ) {
$('input#feedzy_mark_duplicate').attr( 'disabled', false );
} else {
$('input#feedzy_mark_duplicate').attr( 'disabled', true );
}
} );

$(document).on( 'input', 'input[name="custom_vars_value[]"]', function () {
$(this)
.next('.fz-action-icon')
Expand Down
4 changes: 4 additions & 0 deletions tests/e2e/specs/upsell.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ test.describe( 'Upsell', () => {
await page.locator('.fz-form-group:has( #fz-event-execution )').hover({ force: true });
upgradeAlert = page.locator('#feedzy-import-form a[href*="utm_campaign=schedule-import-job"]');
await expect( upgradeAlert ).toBeVisible();

await page.locator('.fz-form-group:has( #feedzy_mark_duplicate )').hover({ force: true });
upgradeAlert = page.locator('#feedzy-import-form a[href*="utm_campaign=remove-duplicates"]');
await expect( upgradeAlert ).toBeVisible();
} );
});

Expand Down
40 changes: 39 additions & 1 deletion tests/test-import.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public function setUp(): void {
* @access public
* @dataProvider importDataProvider
*/
public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic_tags = '[#item_content]', $use_filter = false, $type = 'post' ) {
public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic_tags = '[#item_content]', $use_filter = false, $type = 'post', $check_duplicate = false ) {
do_action( 'init' );

$num_items = $this->import_limit;
Expand Down Expand Up @@ -93,6 +93,11 @@ public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic
$_POST['custom_vars_key'] = array();
$_POST['custom_vars_value'] = array();

if ( $check_duplicate ) {
$_POST['feedzy_meta_data']['import_remove_duplicates'] = 'yes';
$_POST['feedzy_meta_data']['mark_duplicate_tag'] = '[#item_author]';
}

do_action( 'save_post_feedzy_imports', $p->ID, $p );
$this->assertEquals( $p->post_title, $random_name2 );
$this->assertEquals( $p->post_type, 'feedzy_imports' );
Expand All @@ -110,6 +115,11 @@ public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic
$import_custom_fields = get_post_meta( $p->ID, 'imports_custom_fields', true );
$import_feed_limit = get_post_meta( $p->ID, 'import_feed_limit', true );

if ( $check_duplicate ) {
$remove_duplicates = get_post_meta( $p->ID, 'import_remove_duplicates', true );
$mark_duplicate_tag = get_post_meta( $p->ID, 'mark_duplicate_tag', true );
}

$this->assertEquals( $type, $import_post_type );
$this->assertEquals( 'category_' . $category_id, $import_post_term );
$this->assertEquals( 'publish', $import_post_status );
Expand All @@ -123,6 +133,11 @@ public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic
$this->assertEquals( '', $import_custom_fields );
$this->assertEquals( $num_items, $import_feed_limit );

if ( $check_duplicate ) {
$this->assertEquals( 'yes', $remove_duplicates );
$this->assertEquals( '[#item_author]', $mark_duplicate_tag );
}

$feed_src = str_replace( PHP_EOL, '', $urls );

// Do Cron
Expand Down Expand Up @@ -232,6 +247,11 @@ public function test_feedzy_imports( $random_name1, $random_name2, $urls, $magic
$this->assertEquals( 1, get_post_meta( $created[0]->ID, 'feedzy', true ) );
$this->assertNotEmpty( get_post_meta( $created[0]->ID, 'feedzy_item_url', true ) );

// Check found duplicates items.
if ( $check_duplicate ) {
$this->assertNotEmpty( get_post_meta( $created[0]->ID, 'feedzy_mark_duplicate', true ) );
}

// Check Post Delete
$this->assertNotEquals( false, wp_delete_post( $p->ID ) );
return $created[0];
Expand Down Expand Up @@ -370,6 +390,24 @@ function( $date ) {
update_option( 'gmt_offset', 0 ); // reset UTC
}

/**
* Test check duplicate.
*/
public function test_check_duplicate() {
define( 'FEEDZY_PRO_ABSPATH', '' );
add_filter(
'product_feedzy_license_status',
function () {
return 'valid';
},
99
);

$post = $this->test_feedzy_imports( $this->get_rand_name(), $this->get_rand_name(), $this->get_two_rand_feeds(), '[#item_content]', false, 'post', true );

$this->assertNotEmpty( $post );
}

/**
* Utility method to generate a random 5 char string.
*
Expand Down
Loading