tmdinosaurcenter
/
tainacan
mirror of https://github.com/tainacan/tainacan
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #512 from tainacan/feature/457 

Improved efficiency of imported and exported CSV
This commit is contained in:
Vinícius Nunes Medeiros 2021-03-23 15:20:14 -03:00 committed by GitHub
parent 3d35ef0ed7 e51f37d476
commit d6fedad073
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 58 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/classes/class-tainacan-background-process.php
View File

@ -279,17 +279,15 @@ abstract class Background_Process extends \Tainacan_WP_Background_Process {
*/
protected function handle() {
$this->lock_process();
// while we are debugging performance
$newRequest = true;
$batch = $this->get_batch();
if ($newRequest) {
$this->write_log($batch->key, ['New Request']);
$newRequest = false;
if($this->process_lock_in_time != get_site_transient( $this->identifier . '_process_lock' )) {
$this->write_log($batch->key, ['New request has ignored']);
wp_die();
}
$this->write_log($batch->key, ['New Request']);
register_shutdown_function(function() use($batch) {
$error = error_get_last();

43
src/classes/exporter/class-tainacan-exporter.php
View File

@ -193,7 +193,8 @@ abstract class Exporter {
$current_collection_item = $this->get_current_collection_item();
$collections = $this->get_collections();
$collection = $collections[$current_collection];
$current_collection_item ++;
$length_items = $this->get_step_length_items();
$current_collection_item += $length_items;
$this->set_current_collection_item($current_collection_item);
if ($current_collection_item >= $collection['total_items']) {
return $this->next_collection();
@ -248,6 +249,10 @@ abstract class Exporter {
return $this->current_collection_item;
}
public function get_step_length_items() {
return apply_filters('exporter_step_length_items', 20, $this->get_current_step());
}
public function set_current_collection_item($value) {
$this->current_collection_item = $value;
}
@ -350,11 +355,13 @@ abstract class Exporter {
}
public function add_log($message ) {
$this->log[] = $message;
$date_key = '[' . date("Y-m-d H:i:s") . '] ';
$this->log[$date_key] = $message;
}
public function add_error_log($message ) {
$this->error_log[] = $message;
$date_key = '[' . date("Y-m-d H:i:s") . '] ';
$this->error_log[$date_key] = $message;
}
public function is_finished() {
@ -562,20 +569,19 @@ abstract class Exporter {
$this->add_error_log('Collection misconfigured');
return false;
}
$this->add_log('Processing item ' . $current_collection_item);
$length_items = $this->get_step_length_items();
$this->add_log("Processing batch with $length_items items, starting from item $current_collection_item");
$this->process_header($current_collection_item, $collection_definition);
$init = microtime(true);
$processed_items = $this->get_items($current_collection_item, $collection_definition);
foreach ($processed_items as $processed_item) {
$init = microtime(true);
$this->process_item( $processed_item['item'], $processed_item['metadata'] );
$final = microtime(true);
$total = ($final - $init);
$time_log = sprintf( __('Processed in %f seconds', 'tainacan'), $total );
$this->add_log($time_log);
}
$final = microtime(true);
$total = ($final - $init);
$time_log = sprintf( __('Processed in %f seconds', 'tainacan'), $total );
$this->add_log($time_log);
$this->process_footer($current_collection_item, $collection_definition);
return $this->next_item();
@ -595,7 +601,7 @@ abstract class Exporter {
}
private function process_footer($current_collection_item, $collection_definition) {
if ($current_collection_item == $collection_definition['total_items']-1) {
if ($current_collection_item > $collection_definition['total_items']) {
$this->output_footer();
}
}
@ -608,17 +614,18 @@ abstract class Exporter {
private function get_items($index, $collection_definition) {
$collection_id = $collection_definition['id'];
$tainacan_items = \Tainacan\Repositories\Items::get_instance();
$per_page = $this->get_step_length_items();
$page = intdiv($index, $per_page) + 1;
$filters = [
'posts_per_page' => 1,
'paged' => $index+1,
'posts_per_page' => $per_page,
'paged' => $page,
'order' => 'DESC',
'orderby' => 'ID'
];
$this->add_log('Proccessing item index ' . $index . ' in collection ' . $collection_definition['id'] );
$this->add_log("Retrieving $per_page items on page index: $index, in collection " . $collection_definition['id'] );
$items = $tainacan_items->fetch($filters, $collection_id, 'WP_Query');
if ( !isset($collection_definition['total_items']) ) {
$collection_definition['total_items'] = $items->found_posts;
$this->update_current_collection($collection_definition);
@ -805,7 +812,7 @@ abstract class Exporter {
if (method_exists($this, $method_name)) {
$author = $this->get_transient('author');
$this->add_log('User in process: ' . $author . ' (' . date("Y-m-d H:i:s") . ')');
$this->add_log('User in process: ' . $author);
wp_set_current_user($author);
$result = $this->$method_name();
} else {

8
src/classes/generic-background-process/class-tainacan-generic-process.php
View File

@ -146,11 +146,13 @@ abstract class Generic_Process {
}
public function add_log($message ) {
$this->log[] = $message;
$date_key = '[' . date("Y-m-d H:i:s") . '] ';
$this->log[$date_key] = $message;
}
public function add_error_log($message ) {
$this->error_log[] = $message;
public function add_error_log($message ){
$date_key = '[' . date("Y-m-d H:i:s") . '] ';
$this->error_log[$date_key] = $message;
}
public function is_finished() {

10
src/classes/importer/class-tainacan-csv.php
View File

@ -677,14 +677,13 @@ class CSV extends Importer {
$Tainacan_Item_Metadata = \Tainacan\Repositories\Item_Metadata::get_instance();
$Tainacan_Items = \Tainacan\Repositories\Items::get_instance();
// $Tainacan_Items->disable_logs();
// $Tainacan_Metadata->disable_logs();
// $Tainacan_Item_Metadata->disable_logs();
$itemMetadataArray = [];
$updating_item = false;
$Tainacan_Items->disable_logs();
$Tainacan_Metadata->disable_logs();
$Tainacan_Item_Metadata->disable_logs();
if ( is_numeric($this->get_transient('item_id')) ) {
$item = $Tainacan_Items->fetch( (int) $this->get_transient('item_id') );
@ -787,7 +786,8 @@ class CSV extends Importer {
$this->add_error_log( $item->get_errors() );
return false;
}
global $wpdb;
$wpdb->query( 'SET autocommit = 0;' );
foreach ( $itemMetadataArray as $itemMetadata ) {
if($itemMetadata instanceof Entities\Item_Metadata_Entity ) {
$itemMetadata->set_item( $insertedItem ); // *I told you
@ -826,6 +826,8 @@ class CSV extends Importer {
// $this->add_error_log( 'Item ' . $insertedItem->get_id() . ' has an error' );
//}
}
$wpdb->query( 'COMMIT;' );
$wpdb->query( 'SET autocommit = 1;' );
if ( ! $updating_item ) {
$insertedItem->set_status('publish' );

9
src/classes/importer/class-tainacan-importer.php
View File

@ -344,11 +344,13 @@ abstract class Importer {
* @param $type
* @param $messagelog
*/
public function add_log($message ){
$this->log[] = $message;
public function add_log($message ) {
$date_key = '[' . date("Y-m-d H:i:s") . '] ';
$this->log[$date_key] = $message;
}
public function add_error_log($message ){
$this->error_log[] = $message;
$date_key = '[' . date("Y-m-d H:i:s") . '] ';
$this->error_log[$date_key] = $message;
}
public function add_collection(array $collection) {
@ -893,7 +895,6 @@ abstract class Importer {
$author = $this->get_transient('author');
$this->add_log('---------------------------');
$this->add_log('Starting processing new item');
$this->add_log(date("Y-m-d H:i:s"));
$this->add_log('User in process: ' . $author);
wp_set_current_user($author);
$result = $this->$method_name();

8
src/classes/libs/wp-background-process.php
View File

@ -226,6 +226,7 @@
$this->debug('process already running');
return true;
}
$this->debug('process not already running');
return false;
}
@ -241,10 +242,11 @@
$this->debug('locking process: ' . $this->identifier);
$this->start_time = time(); // Set start time of current process.
$max_execution_time = ini_get('max_execution_time');
$lock_duration = ( property_exists( $this, 'queue_lock_time' ) ) ? $this->queue_lock_time : ( empty($max_execution_time) ? 60 : $max_execution_time ); // 1 minute
$lock_duration = ( property_exists( $this, 'queue_lock_time' ) ) ? $this->queue_lock_time : ( empty($max_execution_time) ? 60 : ($max_execution_time * 1.5) ); // 1 minute
$lock_duration = apply_filters( $this->identifier . '_queue_lock_time', $lock_duration );
set_site_transient( $this->identifier . '_process_lock', microtime(), $lock_duration );
$this->process_lock_in_time = microtime();
if(!$this->is_process_running())
set_site_transient( $this->identifier . '_process_lock', $this->process_lock_in_time, $lock_duration );
}
/**

12
src/classes/repositories/class-tainacan-logs.php
View File

@ -260,6 +260,10 @@ class Logs extends Repository {
$collection_id = method_exists($entity, 'get_collection_id') ? $entity->get_collection_id() : 'default';
if ( method_exists($entity, 'get_repository') && !$entity->get_repository()->use_logs ) {
return;
}
if ( $entity instanceof Entities\Collection ) {
$collection_id = $entity->get_id();
$log->set_title( sprintf( __( 'New file was attached to Collection "%s"', 'tainacan'), $entity->get_name() ) );
@ -272,15 +276,11 @@ class Logs extends Repository {
$object_type = get_class($entity);
$object_id = $entity->get_id();
$diff = [];
$log->set_collection_id($collection_id);
$log->set_object_type($object_type);
$log->set_object_id($object_id);
$log->set_action('new-attachment');
$title = __( sprintf('') , 'tainacan');
$prepared = [
'id' => $attachment->ID,
'title' => $attachment->post_title,
@ -313,7 +313,9 @@ class Logs extends Repository {
$entity = Repository::get_entity_by_post( $entity_post );
if ( $entity ) {
if ( method_exists($entity, 'get_repository') && !$entity->get_repository()->use_logs ) {
return;
}
$collection_id = method_exists($entity, 'get_collection_id') ? $entity->get_collection_id() : 'default';
$log = new Entities\Log();