Refactor Importer Class
This commit is contained in:
Leo Germani
parent
d54f21f9c3
commit
3653930f9a
|
|
@ -5,7 +5,11 @@ use Tainacan;
|
|||
|
||||
class CSV extends Importer {
|
||||
|
||||
public function __construct() {
|
||||
protected $manual_mapping = true;
|
||||
|
||||
protected $manual_collection = true;
|
||||
|
||||
public function __construct() {
|
||||
parent::__construct();
|
||||
|
||||
$this->set_default_options([
|
||||
|
|
@ -17,7 +21,7 @@ class CSV extends Importer {
|
|||
/**
|
||||
* @inheritdoc
|
||||
*/
|
||||
public function get_fields(){
|
||||
public function get_source_fields(){
|
||||
$file = new \SplFileObject( $this->tmp_file, 'r' );
|
||||
$file->seek(0 );
|
||||
return $file->fgetcsv( $this->get_option('delimiter') );
|
||||
|
|
@ -27,9 +31,9 @@ class CSV extends Importer {
|
|||
/**
|
||||
* @inheritdoc
|
||||
*/
|
||||
public function process_item( $index ){
|
||||
public function process_item( $index, $collection_index ){
|
||||
$processedItem = [];
|
||||
$headers = $this->get_fields();
|
||||
$headers = $this->get_source_fields();
|
||||
|
||||
// search the index in the file and get values
|
||||
$file = new \SplFileObject( $this->tmp_file, 'r' );
|
||||
|
|
@ -51,13 +55,15 @@ class CSV extends Importer {
|
|||
$processedItem[ $header ] = $values[ $index ];
|
||||
}
|
||||
|
||||
$this->set_progress_current($index+1);
|
||||
|
||||
return $processedItem;
|
||||
}
|
||||
|
||||
/**
|
||||
* @inheritdoc
|
||||
*/
|
||||
public function get_total_items_from_source(){
|
||||
public function get_progress_total_from_source(){
|
||||
$file = new \SplFileObject( $this->tmp_file, 'r' );
|
||||
$file->seek(PHP_INT_MAX);
|
||||
// -1 removing header
|
||||
|
|
|
|||
|
|
@ -5,87 +5,129 @@ use Tainacan\Entities;
|
|||
|
||||
abstract class Importer {
|
||||
|
||||
private $id;
|
||||
private $processed_items = [];
|
||||
|
||||
/**
|
||||
* indicates wether this importer will create all the fields collection and set the mapping
|
||||
* without user interaction
|
||||
*
|
||||
* if set to true, user will have the ability to choose to create a new collection upon importing.
|
||||
*
|
||||
* The importer will have to implement the create_fields_and_mapping() method.
|
||||
*
|
||||
* @var bool
|
||||
*/
|
||||
protected $import_structure_and_mapping = false;
|
||||
|
||||
/**
|
||||
* The collection the items are going to be imported to.
|
||||
* The ID for this importer session
|
||||
*
|
||||
* @var \Tainacan\Entities\Collection
|
||||
* When creating a new importer session via API, an id is returned and used to access this
|
||||
* importer instance in the SESSION array
|
||||
*
|
||||
* @var identifier
|
||||
*/
|
||||
public $collection;
|
||||
|
||||
/**
|
||||
* The mapping from the source metadata structure to the Field Ids of the destination collection
|
||||
*
|
||||
* The format is an array where the keys are the field IDs of the destination collection and the
|
||||
* values are the identifier from the source. This coulb be an ID or a string or whatever the importer finds appropriate to http_persistent_handles_clean
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
public $mapping;
|
||||
private $repository_mapping;
|
||||
private $id;
|
||||
|
||||
/**
|
||||
* The path to the temporary file created when user uploads a file
|
||||
* @var string
|
||||
*/
|
||||
public $tmp_file;
|
||||
protected $tmp_file;
|
||||
|
||||
/**
|
||||
* The total number of items to be imported.
|
||||
* @var int
|
||||
*/
|
||||
protected $total_items;
|
||||
|
||||
/**
|
||||
* THe number of items to be processes in each step
|
||||
* @var int
|
||||
*/
|
||||
private $items_per_step = 100;
|
||||
|
||||
/**
|
||||
* The index of the item to start the import in the next step.
|
||||
* Wether Tainacan must present the user with an interface to manually map
|
||||
* the metadata from the source to the target collection.
|
||||
*
|
||||
* (items are imported in a series of steps, via ajax, to avoid timeout)
|
||||
* @var int
|
||||
* If set to true in the child class, it must implement the method
|
||||
* get_source_fields() to return the field found in the source.
|
||||
*
|
||||
* Note that this will only work when importing items to one single collection.
|
||||
* @var bool
|
||||
*/
|
||||
private $start = 0;
|
||||
private $inside_step_pointer = 0;
|
||||
protected $manual_mapping = false;
|
||||
|
||||
/**
|
||||
* The log with everything that happened during the import process. It generates a report afterwards
|
||||
* Wether Tainacan will let the user choose a destination collection.
|
||||
*
|
||||
* If set to true, the API endpoints will handle Collection creation and will assign it to
|
||||
* the importer object using add_collection() method.
|
||||
*
|
||||
* Otherwise, the child importer class must create the collections and add them to the collections property also
|
||||
* using add_collection()
|
||||
*
|
||||
* @var bool
|
||||
*/
|
||||
protected $manual_collection = true;
|
||||
|
||||
|
||||
/**
|
||||
* The total number of iterations to be imported.
|
||||
*
|
||||
* if not possible to calculate, inform 0 (zero) and no progress bar will be displayed.
|
||||
*
|
||||
* @var int
|
||||
*/
|
||||
protected $progress_total;
|
||||
|
||||
protected $progress_current;
|
||||
|
||||
/**
|
||||
* This array holds the structure that the default step 'process_collections' will handle.
|
||||
*
|
||||
* Its an array of the target collections, with their IDs, the total number of items to be imported and the mapping array
|
||||
* from the source structure to the ID of the metadata fields in tainacan
|
||||
*
|
||||
* The format of the map is an array where the keys are the metadata IDs of the destination collection and the
|
||||
* values are the identifier from the source. This could be an ID or a string or whatever the importer finds appropriate to handle
|
||||
*
|
||||
* Example of the structure of this propery for one collection:
|
||||
* 0 => [
|
||||
* 'id' => 12,
|
||||
* 'map' => [
|
||||
* 30 => 'column1'
|
||||
* 31 => 'column2'
|
||||
* ],
|
||||
* 'total_items' => 1234
|
||||
* ],
|
||||
*
|
||||
* use add_collection() and remove_collection() to interact with thiis array.
|
||||
*
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
public $logs = [];
|
||||
protected $collections = [];
|
||||
|
||||
/**
|
||||
* Stores the options for the importer. Each importer might use this property to save
|
||||
* their own specific option
|
||||
* @var array
|
||||
*/
|
||||
private $options = [];
|
||||
|
||||
private $default_options = [];
|
||||
/**
|
||||
* Stores the default options for the importer options
|
||||
* @var array
|
||||
*/
|
||||
protected $default_options = [];
|
||||
|
||||
private $accpets = [
|
||||
'file' => true,
|
||||
'url' => false,
|
||||
];
|
||||
|
||||
private $is_repository = false;
|
||||
|
||||
private $steps = [];
|
||||
/**
|
||||
* Declares what are the steps the importer will run, in the right order.
|
||||
*
|
||||
* By default, there is only one step, and the callback is the process_collections method
|
||||
* that process items for the collections in the collections array.
|
||||
*
|
||||
* Child classes may declare as many steps as they want and can keep this default step to use
|
||||
* this method for import the items. But it is optional.
|
||||
*
|
||||
* @var array
|
||||
*/
|
||||
protected $steps = [
|
||||
[
|
||||
'name' => 'Import Items',
|
||||
'callback' => 'process_collections'
|
||||
]
|
||||
];
|
||||
|
||||
private $current_step = 0;
|
||||
|
||||
private $in_step_count = 0;
|
||||
|
||||
private $current_collection = 0;
|
||||
|
||||
private $current_collection_item = 0;
|
||||
|
||||
private $url = '';
|
||||
|
||||
public function __construct() {
|
||||
|
|
@ -97,6 +139,9 @@ abstract class Importer {
|
|||
$_SESSION['tainacan_importer'][$this->get_id()] = $this;
|
||||
}
|
||||
|
||||
/////////////////////
|
||||
// Getters and setters
|
||||
|
||||
/**
|
||||
* @return string
|
||||
*/
|
||||
|
|
@ -133,82 +178,125 @@ abstract class Importer {
|
|||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array Mapping
|
||||
*/
|
||||
public function get_mapping(){
|
||||
return $this->mapping;
|
||||
public function get_current_step() {
|
||||
return $this->current_step;
|
||||
}
|
||||
|
||||
public function set_current_step($value) {
|
||||
$this->current_step = $value;
|
||||
}
|
||||
|
||||
public function get_in_step_count() {
|
||||
return $this->in_step_count;
|
||||
}
|
||||
|
||||
public function set_in_step_count($value) {
|
||||
$this->in_step_count = $value;
|
||||
}
|
||||
|
||||
public function get_current_collection() {
|
||||
return $this->current_collection;
|
||||
}
|
||||
|
||||
public function set_current_collection($value) {
|
||||
$this->current_collection = $value;
|
||||
}
|
||||
|
||||
public function get_current_collection_item() {
|
||||
return $this->current_collection_item;
|
||||
}
|
||||
|
||||
public function set_current_collection_item($value) {
|
||||
$this->current_collection_item = $value;
|
||||
}
|
||||
|
||||
public function get_tmp_file(){
|
||||
return $this->tmp_file;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array Array with ids inserted in Tainacan
|
||||
*/
|
||||
public function get_processed_items(){
|
||||
return $this->processed_items;
|
||||
public function set_tmp_file($filepath){
|
||||
$this->tmp_file = $filepath;
|
||||
}
|
||||
|
||||
/**
|
||||
* @return array the last index from source
|
||||
public function get_progress_current() {
|
||||
return $this->progress_current;
|
||||
}
|
||||
|
||||
public function set_progress_current($value) {
|
||||
$this->progress_current = $value;
|
||||
}
|
||||
|
||||
public function get_collections() {
|
||||
return $this->collections;
|
||||
}
|
||||
|
||||
public function set_collections($value) {
|
||||
$this->collections = $value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the options for this importer, including default values for options
|
||||
* that were not set yet.
|
||||
* @return array Importer options
|
||||
*/
|
||||
public function get_logs(){
|
||||
return $this->logs;
|
||||
public function get_options() {
|
||||
return array_merge($this->default_options, $this->options);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param Tainacan\Entities\Collection $collection
|
||||
*/
|
||||
public function set_collection( Entities\Collection $collection ){
|
||||
$this->collection = $collection;
|
||||
/**
|
||||
* Set the options array
|
||||
* @param array $options
|
||||
*/
|
||||
public function set_options($options) {
|
||||
$this->options = $options;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the default options values.
|
||||
*
|
||||
* Must be called from the __construct method of the child importer class to set default values.
|
||||
*
|
||||
* @param array $options
|
||||
*/
|
||||
protected function set_default_options($options) {
|
||||
$this->default_options = $options;
|
||||
}
|
||||
|
||||
|
||||
public function set_steps($steps) {
|
||||
$this->steps = $steps;
|
||||
}
|
||||
|
||||
/**
|
||||
* save an associative array with tainacan field id as index and field from source as value
|
||||
public function get_steps() {
|
||||
return $this->steps;
|
||||
}
|
||||
|
||||
/**
|
||||
* return the total progress number to calculate progress
|
||||
*
|
||||
* @param array $mapping Mapping importer-fields
|
||||
* @return int Total of items
|
||||
*/
|
||||
public function set_mapping( $mapping){
|
||||
if(!empty($mapping))
|
||||
{
|
||||
$this->mapping = $mapping;
|
||||
}
|
||||
}
|
||||
public function get_progress_total() {
|
||||
if ( !isset( $this->progress_total ) ) {
|
||||
if ( method_exists($this, 'get_progress_total_from_source') ) {
|
||||
$this->progress_total = $this->get_progress_total_from_source();
|
||||
} else {
|
||||
$this->progress_total = 0;
|
||||
}
|
||||
|
||||
public function set_repository_mapping( $mapping, $item_id ){
|
||||
if(!empty($mapping) && !empty($item_id))
|
||||
{
|
||||
$this->repository_mapping[$item_id] = $mapping;
|
||||
}else return false;
|
||||
}
|
||||
}
|
||||
return $this->progress_total;
|
||||
}
|
||||
|
||||
public function get_repository_mapping($item_id)
|
||||
{
|
||||
if(!empty($item_id))
|
||||
{
|
||||
return $this->repository_mapping[$item_id];
|
||||
}else return false;
|
||||
}
|
||||
////////////////////////////////////
|
||||
// Utilities
|
||||
|
||||
/**
|
||||
* set how many items should be processes in each step
|
||||
*
|
||||
* @param $size The total of items
|
||||
*/
|
||||
public function set_items_per_step( $size ){
|
||||
$this->items_per_step = $size;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param int $start the first index to init the process
|
||||
*/
|
||||
public function set_start( $start ){
|
||||
$this->start = $start;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param $file File to be managed by importer
|
||||
* @return bool
|
||||
*/
|
||||
public function set_file( $file ){
|
||||
public function add_file( $file ){
|
||||
$new_file = $this->upload_file( $file );
|
||||
if ( is_numeric( $new_file ) ) {
|
||||
$this->tmp_file = get_attached_file( $new_file );
|
||||
|
|
@ -217,21 +305,7 @@ abstract class Importer {
|
|||
}
|
||||
}
|
||||
|
||||
public function set_inside_step_pointer($step_pointer)
|
||||
{
|
||||
if(is_numeric($step_pointer) && $step_pointer >= 0)
|
||||
{
|
||||
$this->inside_step_pointer = $step_pointer;
|
||||
}else
|
||||
{
|
||||
$this->inside_step_pointer = 0;
|
||||
}
|
||||
}
|
||||
|
||||
public function get_inside_step_pointer()
|
||||
{
|
||||
return $this->inside_step_pointer;
|
||||
}
|
||||
/**
|
||||
* log the actions from importer
|
||||
*
|
||||
|
|
@ -242,6 +316,22 @@ abstract class Importer {
|
|||
$this->logs[] = [ 'type' => $type, 'message' => $message ];
|
||||
}
|
||||
|
||||
public function add_collection(array $collection) {
|
||||
if (isset($collection['id'])) {
|
||||
$this->remove_collection($collection['id']);
|
||||
$this->collections[] = $collection;
|
||||
}
|
||||
}
|
||||
|
||||
public function remove_collection($col_id) {
|
||||
foreach ($this->get_collections() as $index => $col) {
|
||||
if ($col['id'] == $col_id) {
|
||||
unset($this->collections[$index]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* internal function to upload the file
|
||||
*
|
||||
|
|
@ -268,56 +358,10 @@ abstract class Importer {
|
|||
$file = fopen( $this->get_id().'.txt', 'w' );
|
||||
fwrite( $file, $tmp['body'] );
|
||||
fclose( $file );
|
||||
return $this->set_file( $this->get_id().'.txt' );
|
||||
return $this->add_file( $this->get_id().'.txt' );
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* get the fields of file/url to allow mapping
|
||||
* should return an array
|
||||
*
|
||||
* @return array $fields_source the fields from the source
|
||||
*/
|
||||
abstract public function get_fields();
|
||||
|
||||
/**
|
||||
* get values for a single item
|
||||
*
|
||||
* @param $index
|
||||
* @return array with field_source's as the index and values for the
|
||||
* item
|
||||
*
|
||||
* Ex: [ 'Field1' => 'value1', 'Field2' => [ 'value2','value3' ]
|
||||
*/
|
||||
abstract public function process_item( $index );
|
||||
|
||||
/**
|
||||
* return the all items found
|
||||
*
|
||||
* @return int Total of items
|
||||
*/
|
||||
public function get_total_items() {
|
||||
if ( !isset( $this->total_items ) ) {
|
||||
$this->total_items = $this->get_total_items_from_source();
|
||||
}
|
||||
return $this->total_items;
|
||||
}
|
||||
|
||||
/**
|
||||
* Method implemented by the child importer class to return the number of items to be imported
|
||||
* @return int
|
||||
*/
|
||||
abstract public function get_total_items_from_source();
|
||||
|
||||
/**
|
||||
* Gets the options for this importer, including default values for options
|
||||
* that were not set yet.
|
||||
* @return array Importer options
|
||||
*/
|
||||
public function get_options() {
|
||||
return array_merge($this->default_options, $this->options);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets one option from the options array.
|
||||
*
|
||||
|
|
@ -331,25 +375,6 @@ abstract class Importer {
|
|||
return isset($options[$key]) ? $options[$key] : '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the default options values.
|
||||
*
|
||||
* Must be called from the __construct method of the child importer class to set default values.
|
||||
*
|
||||
* @param array $options
|
||||
*/
|
||||
protected function set_default_options($options) {
|
||||
$this->default_options = $options;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the options array
|
||||
* @param array $options
|
||||
*/
|
||||
public function set_options($options) {
|
||||
$this->options = $options;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new method accepeted by the importer
|
||||
*
|
||||
|
|
@ -366,38 +391,6 @@ abstract class Importer {
|
|||
return false;
|
||||
}
|
||||
|
||||
protected function get_start()
|
||||
{
|
||||
return $this->start;
|
||||
}
|
||||
|
||||
protected function get_items_per_step()
|
||||
{
|
||||
return $this->items_per_step;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets importer as repository importer
|
||||
*/
|
||||
public function set_repository()
|
||||
{
|
||||
$this->is_repository = true;
|
||||
}
|
||||
|
||||
public function set_steps($steps)
|
||||
{
|
||||
$this->steps =$steps;
|
||||
}
|
||||
|
||||
public function is_finished()
|
||||
{
|
||||
if($this->current_step >= count($this->steps))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
/**
|
||||
* Removes method accepeted by the importer
|
||||
*
|
||||
|
|
@ -414,59 +407,163 @@ abstract class Importer {
|
|||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* process a limited size of items
|
||||
*
|
||||
* @param int $start the index of the item to start processing from
|
||||
*/
|
||||
public function process( $start ){
|
||||
|
||||
$end = $start + $this->items_per_step;
|
||||
|
||||
while ( $start < $end && count( $this->get_processed_items() ) < $this->get_total_items() ) {
|
||||
$processed_item = $this->process_item( $start );
|
||||
if( $processed_item) {
|
||||
$this->insert( $start, $processed_item );
|
||||
} else {
|
||||
$this->add_log('error', 'failed on item '.$start );
|
||||
break;
|
||||
}
|
||||
$start++;
|
||||
public function is_finished()
|
||||
{
|
||||
if($this->current_step >= count($this->steps))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
$this->set_start($start);
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
///////////////////////////////
|
||||
// Abstract methods
|
||||
|
||||
|
||||
/**
|
||||
* get the fields of file/url to allow mapping
|
||||
* should return an array
|
||||
*
|
||||
* Used when $manual_mapping is set to true, to build the mapping interface
|
||||
*
|
||||
* @return array $fields_source the fields from the source
|
||||
*/
|
||||
abstract public function get_source_fields();
|
||||
|
||||
/**
|
||||
* get values for a single item
|
||||
*
|
||||
* @param $index
|
||||
* @return array with field_source's as the index and values for the
|
||||
* item
|
||||
*
|
||||
* Ex: [ 'Field1' => 'value1', 'Field2' => [ 'value2','value3' ]
|
||||
*/
|
||||
abstract public function process_item( $index, $collection_id );
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Method implemented by the child importer class to return the total number of interations the importer must run
|
||||
*
|
||||
* Used to build the progress bar
|
||||
*
|
||||
* @return int
|
||||
*/
|
||||
abstract public function get_progress_total_from_source();
|
||||
|
||||
|
||||
|
||||
////////////////////////////////////////
|
||||
// Core methods
|
||||
|
||||
/**
|
||||
* process an item from the collections queue
|
||||
*
|
||||
*/
|
||||
public function process_collections() {
|
||||
|
||||
$current_collection = $this->get_current_collection();
|
||||
$current_collection_item = $this->get_current_collection_item();
|
||||
|
||||
$processed_item = $this->process_item( $current_collection_item, $current_collection );
|
||||
if( $processed_item) {
|
||||
$this->insert( $processed_item, $current_collection );
|
||||
} else {
|
||||
$this->add_log('error', 'failed on item '. $start );
|
||||
}
|
||||
|
||||
return $this->next_item();
|
||||
}
|
||||
|
||||
protected function next_item() {
|
||||
|
||||
$current_collection = $this->get_current_collection();
|
||||
$current_collection_item = $this->get_current_collection_item();
|
||||
$collections = $this->get_collections();
|
||||
$collection = $collections[$current_collection];
|
||||
|
||||
$current_collection_item ++;
|
||||
$this->set_current_collection_item($current_collection_item);
|
||||
|
||||
if ($current_collection_item >= $collection['total_items']) {
|
||||
return $this->next_collection();
|
||||
}
|
||||
|
||||
return $current_collection_item;
|
||||
|
||||
}
|
||||
|
||||
protected function next_collection() {
|
||||
|
||||
$current_collection = $this->get_current_collection();
|
||||
$collections = $this->get_collections();
|
||||
|
||||
$this->set_current_collection_item(0);
|
||||
|
||||
$current_collection ++;
|
||||
|
||||
if (isset($collections[$current_collection])) {
|
||||
$this->set_current_collection($current_collection);
|
||||
return $current_collection;
|
||||
}
|
||||
|
||||
return false;
|
||||
|
||||
|
||||
}
|
||||
|
||||
protected function next_step() {
|
||||
|
||||
$current_step = $this->get_current_step();
|
||||
$steps = $this->get_steps();
|
||||
|
||||
$current_step ++;
|
||||
$this->set_current_step($current_step);
|
||||
|
||||
if (isset($steps[$current_step])) {
|
||||
return $current_step;
|
||||
}
|
||||
|
||||
return false;
|
||||
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* insert processed item from source to Tainacan
|
||||
*
|
||||
* @param int $index the source id unique for the item
|
||||
* @param array $processed_item Associative array with field source's as index with
|
||||
* its value or values
|
||||
* @return Tainacan\Entities\Item Item inserted
|
||||
*/
|
||||
public function insert( $index, $processed_item ){
|
||||
$Tainacan_Fields = \Tainacan\Repositories\Fields::get_instance();
|
||||
public function insert( $processed_item, $collection_index ) {
|
||||
|
||||
$collections = $this->get_collections();
|
||||
$collection_definition = isset($collections[$collection_index]) ? $collections[$collection_index] : false;
|
||||
if ( !$collection_definition || !is_array($collection_definition) || !isset($collection_definition['id']) || !isset($collection_definition['map']) ) {
|
||||
$this->add_log('error','Collection misconfigured');
|
||||
return false;
|
||||
}
|
||||
|
||||
$collection = \Tainacan\Repositories\Collections::get_instance()->fetch($collection_definition['id']);
|
||||
|
||||
$Tainacan_Fields = \Tainacan\Repositories\Fields::get_instance();
|
||||
$Tainacan_Item_Metadata = \Tainacan\Repositories\Item_Metadata::get_instance();
|
||||
$Tainacan_Items = \Tainacan\Repositories\Items::get_instance();
|
||||
|
||||
$isUpdate = ( is_array( $this->processed_items ) && isset( $this->processed_items[ $index ] ) )
|
||||
? $this->processed_items[ $index ] : 0;
|
||||
$item = new Entities\Item( $isUpdate );
|
||||
$item = new Entities\Item();
|
||||
$itemMetadataArray = [];
|
||||
|
||||
if( !isset( $this->mapping ) ){
|
||||
$this->add_log('error','Mapping is not set');
|
||||
return false;
|
||||
}
|
||||
|
||||
if( is_array( $processed_item ) ){
|
||||
foreach ( $processed_item as $field_source => $values ){
|
||||
$tainacan_field_id = array_search( $field_source, $this->mapping );
|
||||
$tainacan_field_id = array_search( $field_source, $collection_definition['map'] );
|
||||
$field = $Tainacan_Fields->fetch( $tainacan_field_id );
|
||||
|
||||
if( $field instanceof Entities\Field ){
|
||||
$singleItemMetadata = new Entities\Item_Metadata_Entity( $item, $field);
|
||||
$singleItemMetadata = new Entities\Item_Metadata_Entity( $item, $field); // *empty item will be replaced by inserted in the next foreach
|
||||
$singleItemMetadata->set_value( $values );
|
||||
$itemMetadataArray[] = $singleItemMetadata;
|
||||
}
|
||||
|
|
@ -474,8 +571,8 @@ abstract class Importer {
|
|||
}
|
||||
}
|
||||
|
||||
if( !empty( $itemMetadataArray ) && $this->collection instanceof Entities\Collection ){
|
||||
$item->set_collection( $this->collection );
|
||||
if( !empty( $itemMetadataArray ) && $collection instanceof Entities\Collection ){
|
||||
$item->set_collection( $collection );
|
||||
|
||||
if( $item->validate() ){
|
||||
$insertedItem = $Tainacan_Items->insert( $item );
|
||||
|
|
@ -485,29 +582,27 @@ abstract class Importer {
|
|||
}
|
||||
|
||||
foreach ( $itemMetadataArray as $itemMetadata ) {
|
||||
$itemMetadata->set_item( $insertedItem );
|
||||
$itemMetadata->set_item( $insertedItem ); // *I told you
|
||||
|
||||
if( $itemMetadata->validate() ){
|
||||
$result = $Tainacan_Item_Metadata->insert( $itemMetadata );
|
||||
} else {
|
||||
$this->add_log( 'error', 'Item ' . $index . ' on field '. $itemMetadata->get_field()->get_name()
|
||||
$this->add_log( 'error', 'Item ' . $insertedItem->get_id() . ' on field '. $itemMetadata->get_field()->get_name()
|
||||
.' has error ' . $itemMetadata->get_errors() );
|
||||
continue;
|
||||
}
|
||||
|
||||
if( $result ){
|
||||
$values = ( is_array( $itemMetadata->get_value() ) ) ? implode( PHP_EOL, $itemMetadata->get_value() ) : $itemMetadata->get_value();
|
||||
$this->add_log( 'success', 'Item ' . $index .
|
||||
$this->add_log( 'success', 'Item ' . $insertedItem->get_id() .
|
||||
' has inserted the values: ' . $values . ' on field: ' . $itemMetadata->get_field()->get_name() );
|
||||
} else {
|
||||
$this->add_log( 'error', 'Item ' . $index . ' has an error' );
|
||||
$this->add_log( 'error', 'Item ' . $insertedItem->get_id() . ' has an error' );
|
||||
}
|
||||
}
|
||||
|
||||
$insertedItem->set_status('publish' );
|
||||
|
||||
// inserted the id on processed item with its index as array index
|
||||
$this->processed_items[ $index ] = $insertedItem->get_id();
|
||||
if($insertedItem->validate()) {
|
||||
$insertedItem = $Tainacan_Items->update( $insertedItem );
|
||||
} else {
|
||||
|
|
@ -516,6 +611,7 @@ abstract class Importer {
|
|||
}
|
||||
|
||||
return $insertedItem;
|
||||
|
||||
} else {
|
||||
$this->add_log( 'error', 'Collection not set');
|
||||
return false;
|
||||
|
|
@ -524,47 +620,36 @@ abstract class Importer {
|
|||
}
|
||||
|
||||
/**
|
||||
* run the process
|
||||
* runs one iteration
|
||||
*/
|
||||
public function run(){
|
||||
if($this->is_repository && $this->current_step < count($this->steps))
|
||||
{
|
||||
//$process_name = key($this->steps);
|
||||
$function_name = current($this->steps);
|
||||
$inside_step_pointer = $this->{$function_name}();//If unlike numeric this means that still there is stuff to process
|
||||
|
||||
if($inside_step_pointer === false || (!is_numeric($inside_step_pointer) || $inside_step_pointer < 0))
|
||||
{
|
||||
//Move on to the next step
|
||||
next($this->steps);
|
||||
$this->current_step++;
|
||||
$this->set_inside_step_pointer(0);
|
||||
}else if(is_numeric($inside_step_pointer) && $inside_step_pointer > 0)
|
||||
{
|
||||
$this->set_inside_step_pointer($inside_step_pointer);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if ( ( !isset($this->collection) || ! $this->collection instanceof Entities\Collection ) && $this->import_structure_and_mapping ) {
|
||||
$new_collection = new Entities\Collection();
|
||||
$new_collection->set_name('New Imported Collection');
|
||||
$new_collection->set_status('publish');
|
||||
$new_collection->validate();
|
||||
$new_collection = Tainacan\Repositories\Collections::get_instance()->insert($new_collection);
|
||||
if ($this->is_finished()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$this->set_collection($new_collection);
|
||||
$steps = $this->get_steps();
|
||||
$current_step = $this->get_current_step();
|
||||
$method_name = $steps[$current_step]['callback'];
|
||||
|
||||
if (!method_exists($this, 'create_fields_and_mapping')) {
|
||||
throw new Exception('Importers with import_structure_and_mapping true must implement create_fields_and_mapping method');
|
||||
}
|
||||
if (method_exists($this, $method_name)) {
|
||||
$result = $this->$method_name();
|
||||
} else {
|
||||
$this->add_log( 'error', 'Callback not found for step ' . $steps[$current_step]['name']);
|
||||
$result = false;
|
||||
}
|
||||
|
||||
$this->create_fields_and_mapping();
|
||||
if($result === false || (!is_numeric($result) || $result < 0)) {
|
||||
//Move on to the next step
|
||||
$this->set_in_step_count(0);
|
||||
$return = $this->next_step();
|
||||
} else if(is_numeric($result) && $result > 0) {
|
||||
$this->set_in_step_count($result);
|
||||
$return = $result;
|
||||
}
|
||||
|
||||
return $return;
|
||||
|
||||
}
|
||||
|
||||
$this->process( $this->start );
|
||||
return sizeof($this->get_processed_items());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -11,7 +11,7 @@ use Tainacan\Importer;
|
|||
|
||||
class ImporterTests extends TAINACAN_UnitTestCase {
|
||||
|
||||
public function test_intance_old_tainacan()
|
||||
/*public function test_intance_old_tainacan()
|
||||
{
|
||||
$collection = $this->tainacan_entity_factory->create_entity(
|
||||
'collection',
|
||||
|
|
@ -27,10 +27,10 @@ class ImporterTests extends TAINACAN_UnitTestCase {
|
|||
$old_tainacan_importer = new Importer\Old_Tainacan();
|
||||
$id = $old_tainacan_importer->get_id();
|
||||
$_SESSION['tainacan_importer'][$id]->set_collection( $collection );
|
||||
$this->assertEquals( $collection->get_id(), $_SESSION['tainacan_importer'][$id]->collection->get_id() );
|
||||
this->assertEquals( $collection->get_id(), $_SESSION['tainacan_importer'][$id]->collection->get_id() );
|
||||
}
|
||||
|
||||
/*public function test_automapping_old_tainacan()
|
||||
public function test_automapping_old_tainacan()
|
||||
{
|
||||
//$Tainacan_Items = \Tainacan\Repositories\Items::get_instance();
|
||||
//$Tainacan_Fields = \Tainacan\Repositories\Fields::get_instance();
|
||||
|
|
@ -134,29 +134,6 @@ class ImporterTests extends TAINACAN_UnitTestCase {
|
|||
|
||||
$this->assertEquals( $_SESSION['tainacan_importer'][$id]->get_total_items(), count( $items ) );
|
||||
}*/
|
||||
/**
|
||||
* @group importer
|
||||
*/
|
||||
public function test_instance_csv () {
|
||||
|
||||
$collection = $this->tainacan_entity_factory->create_entity(
|
||||
'collection',
|
||||
array(
|
||||
'name' => 'Other',
|
||||
'description' => 'adasdasdsa',
|
||||
'default_order' => 'DESC',
|
||||
'status' => 'publish'
|
||||
),
|
||||
true
|
||||
);
|
||||
|
||||
$csv_importer = new Importer\CSV();
|
||||
$id = $csv_importer->get_id();
|
||||
$_SESSION['tainacan_importer'][$id]->set_collection( $collection );
|
||||
|
||||
// here the session is init already
|
||||
$this->assertEquals( $collection->get_id(), $_SESSION['tainacan_importer'][$id]->collection->get_id() );
|
||||
}
|
||||
|
||||
/**
|
||||
* @group importer
|
||||
|
|
@ -168,8 +145,6 @@ class ImporterTests extends TAINACAN_UnitTestCase {
|
|||
$csv_importer = new Importer\CSV();
|
||||
$id = $csv_importer->get_id();
|
||||
|
||||
$_SESSION['tainacan_importer'][$id]->set_items_per_step(2);
|
||||
|
||||
// open the file "demosaved.csv" for writing
|
||||
$file = fopen($file_name, 'w');
|
||||
|
||||
|
|
@ -193,21 +168,16 @@ class ImporterTests extends TAINACAN_UnitTestCase {
|
|||
// Close the file
|
||||
fclose($file);
|
||||
|
||||
$_SESSION['tainacan_importer'][$id]->set_file( $file_name );
|
||||
|
||||
if(!isset( $_SESSION['tainacan_importer'][$id]->tmp_file )){
|
||||
#TODO: Remove dependence of web server (see fetch_from_remote)
|
||||
$this->markTestSkipped('This test need a apache installation available.');
|
||||
}
|
||||
$_SESSION['tainacan_importer'][$id]->set_tmp_file( $file_name );
|
||||
|
||||
// file isset on importer
|
||||
$this->assertTrue( isset( $_SESSION['tainacan_importer'][$id]->tmp_file ) );
|
||||
$this->assertTrue( !empty( $_SESSION['tainacan_importer'][$id]->get_tmp_file() ) );
|
||||
|
||||
// count size of csv
|
||||
$this->assertEquals( 5, $_SESSION['tainacan_importer'][$id]->get_total_items() );
|
||||
$this->assertEquals( 5, $_SESSION['tainacan_importer'][$id]->get_progress_total_from_source() );
|
||||
|
||||
// get fields to mapping
|
||||
$headers = $_SESSION['tainacan_importer'][$id]->get_fields();
|
||||
$headers = $_SESSION['tainacan_importer'][$id]->get_source_fields();
|
||||
$this->assertEquals( $headers[4], 'Column 5' );
|
||||
|
||||
// inserting the collection
|
||||
|
|
@ -222,8 +192,10 @@ class ImporterTests extends TAINACAN_UnitTestCase {
|
|||
true
|
||||
);
|
||||
|
||||
// set the importer
|
||||
$_SESSION['tainacan_importer'][$id]->set_collection( $collection );
|
||||
$collection_definition = [
|
||||
'id' => $collection->get_id(),
|
||||
'total_items' => $_SESSION['tainacan_importer'][$id]->get_progress_total_from_source(),
|
||||
];
|
||||
|
||||
// get collection fields to map
|
||||
$fields = $Tainacan_Fields->fetch_by_collection( $collection, [], 'OBJECT' ) ;
|
||||
|
|
@ -234,28 +206,29 @@ class ImporterTests extends TAINACAN_UnitTestCase {
|
|||
$map[$field->get_id()] = $headers[$index];
|
||||
}
|
||||
|
||||
// set the mapping
|
||||
$_SESSION['tainacan_importer'][$id]->set_mapping( $map );
|
||||
$collection_definition['map'] = $map;
|
||||
|
||||
// check is equal
|
||||
$this->assertEquals( $_SESSION['tainacan_importer'][$id]->get_mapping(), $map );
|
||||
// add the collection
|
||||
$_SESSION['tainacan_importer'][$id]->add_collection( $collection_definition );
|
||||
|
||||
//execute the process
|
||||
|
||||
$this->assertEquals(2, $_SESSION['tainacan_importer'][$id]->run(), 'first step should import 2 items');
|
||||
$this->assertEquals(4, $_SESSION['tainacan_importer'][$id]->run(), 'second step should import 2 items');
|
||||
$this->assertEquals(5, $_SESSION['tainacan_importer'][$id]->run(), 'third step should import 3 items');
|
||||
$this->assertEquals(5, $_SESSION['tainacan_importer'][$id]->run(), 'if call run again after finish, do nothing');
|
||||
$this->assertEquals(1, $_SESSION['tainacan_importer'][$id]->run(), 'first step should import 1 item');
|
||||
$this->assertEquals(2, $_SESSION['tainacan_importer'][$id]->run(), 'second step should import 2 items');
|
||||
$this->assertEquals(3, $_SESSION['tainacan_importer'][$id]->run(), 'third step should import 3 items');
|
||||
$this->assertEquals(4, $_SESSION['tainacan_importer'][$id]->run(), 'third step should import 4 items');
|
||||
$this->assertEquals(false, $_SESSION['tainacan_importer'][$id]->run(), '5 items and return false because its finished');
|
||||
$this->assertEquals(false, $_SESSION['tainacan_importer'][$id]->run(), 'if call run again after finish, do nothing');
|
||||
|
||||
$items = $Tainacan_Items->fetch( [], $collection, 'OBJECT' );
|
||||
|
||||
$this->assertEquals( $_SESSION['tainacan_importer'][$id]->get_total_items(), count( $items ) );
|
||||
$this->assertEquals( $_SESSION['tainacan_importer'][$id]->get_progress_total_from_source(), count( $items ) );
|
||||
}
|
||||
|
||||
/**
|
||||
* @group importer
|
||||
*/
|
||||
public function test_fetch_file(){
|
||||
/*
|
||||
public function test_fetch_file(){
|
||||
$csv_importer = new Importer\CSV();
|
||||
$id = $csv_importer->get_id();
|
||||
$_SESSION['tainacan_importer'][$id]->fetch_from_remote( 'http://localhost/wordpress-test/wp-json' );
|
||||
|
|
@ -267,4 +240,5 @@ class ImporterTests extends TAINACAN_UnitTestCase {
|
|||
|
||||
$this->assertTrue( isset( $_SESSION['tainacan_importer'][$id]->tmp_file ) );
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue