Do not stop queuing in import users

[BehroozBvk]

Hi,I cite this topic: #2304 I have a problem.
I used the above code to import users using, WithChunkReading, WithBatchInserts, ShouldQueue
But instead of sending the notification, I save the validation errors in an Excel file for the user to download later.

So far it works properly but the queue does not stop and it is processed and processed in a row even for a long time until the memory fills up and the system stops.

What is the problem?
@Naoray @patrickbrouwers

ToCollectionImport.php

abstract class ToCollectionImport implements ToCollection
{
    /**
     * Set additional user resolve callback.
     *
     * @var Collection
     */
    protected static Collection $additionalUsersCallback;

    abstract public function processImport(Collection $rows);

    abstract public function rules(): array;

    abstract public function getUser();

    abstract public function getOrganization();

    /**
     * @param Collection $rows
     * @throws ValidationException|Throwable
     */
    public function collection(Collection $rows)
    {
        if ($this instanceof WithValidation) {
            $rows = $this->validate($rows);
        }

        try {
            $this->processImport($rows);
        } catch (Throwable $e) {
            $this->recordOrThrowErrors($e);
        }

        if ($this->failures()->count() > 0) {
            $path = "excels/userFailures/{$this->getOrganization()}/import_failures_" . now()->format('Y-m-d H:i:s') . '.xlsx';
            (new UsersFailuresImportedExport($this->failures()))->store($path);
        }

        if ($this->errors()->count() > 0) {
            Log::error($this->errors());
        }
    }

    /**
     * Validate given collection data.
     *
     * @param Collection $rows
     *
     * @return Collection
     * @throws ValidationException
     *
     */
    protected function validate(Collection $rows): Collection
    {
        $validator = Validator::make($rows->toArray(), $this->rules());

        if (!$validator->fails()) {
            return $rows;
        }

        if ($this instanceof SkipsOnFailure) {
            $this->onFailure(
                ...$this->collectErrors($validator, $rows)
            );

            $keysCausingFailure = collect($validator->errors()->keys())->map(function ($key) {
                return Str::before($key, '.');
            })->values()->toArray();

            return $rows->except($keysCausingFailure);
        }

        throw new ValidationException($validator);
    }

    /**
     * Get all validation errors.
     *
     * @param $validator
     * @param Collection $rows
     *
     * @return array
     */
    protected function collectErrors($validator, Collection $rows): array
    {
        $failures = [];

        foreach ($validator->errors() as $attribute => $messages) {
            $row = strtok($attribute, '.');
            $attributeName = strtok('');
            $attributeName = $attributes['*.' . $attributeName] ?? $attributeName;

            $failures[] = new Failure(
                $row,
                $attributeName,
                str_replace($attribute, $attributeName, $messages),
                $rows[$row] ?? []
            );
        }


        return $failures;
    }

    /**
     * Records an error or throws its exception.
     *
     * @param Throwable $error
     *
     * @return void
     * @throws Throwable
     */
    protected function recordOrThrowErrors(Throwable $error)
    {
        if ($this instanceof SkipsOnError) {
            return $this->onError($error);
        }

        throw $error;
    }

    /**
     * Set additional users callback.
     *
     * @param Callable $callback
     *
     * @return self
     */
    public static function additionalUsers(callable $callback): ToCollectionImport
    {
        static::$additionalUsersCallback = $callback;
    }
}

UserImport.php

class UsersImport extends ToCollectionImport implements SkipsOnError
    , SkipsOnFailure
    , SkipsEmptyRows
    , WithValidation
    , WithStartRow
    , WithHeadingRow
    , WithChunkReading
    , WithBatchInserts
    , ShouldQueue
{
    use Importable, SkipsErrors, SkipsFailures;

    /**
     * skip heading row and start next row.
     * @return int
     */
    public function startRow(): int
    {
        return 2;
    }

    /**
     * @return string[][]
     */
    public function rules(): array
    {
        return [
            '*.first_name' => ['required'],
            '*.last_name' => ['required'],
            '*.username' => ['required', 'unique:users,username'],
            '*.password' => ['required'],
            '*.mobile' => ['unique:users,mobile'],
            '*.email' => ['email', 'unique:users,email'],
        ];
    }

    /**
     * @return int
     */
    public function batchSize(): int
    {
        return 500;
    }

    /**
     * @return int
     */
    public function chunkSize(): int
    {
        return 500;
    }

    /**
     * @param Collection $rows
     * @return User
     */
    public function processImport(Collection $rows): User
    {
        $role = Role::findByName('normal-user');
        foreach ($rows as $row) {
            $user = User::create([
                'first_name' => $row['first_name'],
                'last_name' => $row['last_name'],
                'username' => $row['username'],
                'password' => Hash::make($row['password']),
                'mobile' => $row['mobile'],
                'email' => $row['email'],
            ]);
            if ($user) {
                //Todo: set organization from session.
                //Todo: set group user.
                $user->assignRole([$role->id => ['organization_id' => Organization::DEFAULT]]);
            }
        }
    }

    public function getUser()
    {
        return 125;
    }

    public function getOrganization()
    {
        return 1;
    }
}

UsersFailuresImportedExport.php

<?php

namespace App\Exports;

use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Log;
use Maatwebsite\Excel\Concerns\Exportable;
use Maatwebsite\Excel\Concerns\FromCollection;
use Maatwebsite\Excel\Concerns\ShouldAutoSize;
use Maatwebsite\Excel\Concerns\WithEvents;
use Maatwebsite\Excel\Concerns\WithHeadings;
use Maatwebsite\Excel\Events\AfterSheet;
use PhpOffice\PhpSpreadsheet\Style\Alignment;

class UsersFailuresImportedExport implements FromCollection,
    ShouldAutoSize,
    WithEvents,
    WithHeadings,
    ShouldQueue
{
    use Exportable;

    private int $row_count;
    protected Collection $failures;

    public function __construct(Collection $failures)
    {
        $getAllFailures = $failures->map(function ($item) {
            return ['row' => $item->row(), 'attribute' => $item->attribute(), 'errors' => $item->errors()];
        })->groupBy('row');
        $errors = [];
        $collection = new Collection();
        foreach ($getAllFailures->toArray() as $row => $error) {
            foreach ($error as $e) {
                $errors[$row][] = str_replace($e['attribute'], trans("validation.attributes.{$e['attribute']}"), $e['errors'][0]);
            }
            $collection->push((object)[$row, implode("\r\n", $errors[$row])]);
        }
        $this->row_count = $collection->count() + 1;
        $this->failures = $collection;

    }

    public function collection(): Collection
    {
        return $this->failures;
    }

    public function registerEvents(): array
    {
        $rowCount = $this->row_count;
        Log::info($rowCount);
        return [
            AfterSheet::class => function (AfterSheet $event) use($rowCount) {
                $event->sheet->getDelegate()->setRightToLeft(true)
                    ->getStyle("A1:B{$rowCount}")
                    ->getAlignment()
                    ->setHorizontal(Alignment::HORIZONTAL_CENTER)
                    ->setVertical(Alignment::VERTICAL_CENTER);
                $event->sheet->styleCells(
                    "A1:B1",
                    [
                        'font' => [
                            'bold' => true,
                            'color' => ['rgb' => 'ffffff']
                        ]
                    ]
                );
                $event->sheet->getDelegate()->getStyle('A1:B1')->getFill()
                    ->setFillType(\PhpOffice\PhpSpreadsheet\Style\Fill::FILL_SOLID)
                    ->getStartColor()->setRGB('f44336');
            }
        ];
    }

    public function headings(): array
    {
        return [
            'row',
            'errors'
        ];
    }
}

Log Queues: Repeat ReadChunk

[2021-08-06 23:05:55][4371] Processing: Maatwebsite\Excel\Jobs\QueueImport
[2021-08-06 23:05:55][4371] Processed:  Maatwebsite\Excel\Jobs\QueueImport
[2021-08-06 23:05:55][4372] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:05:58][4372] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:05:58][4373] Processing: Maatwebsite\Excel\Jobs\QueueExport
[2021-08-06 23:05:58][4373] Processed:  Maatwebsite\Excel\Jobs\QueueExport
[2021-08-06 23:05:58][4374] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:00][4375] Processing: Maatwebsite\Excel\Jobs\AppendDataToSheet
[2021-08-06 23:06:01][4375] Processed:  Maatwebsite\Excel\Jobs\AppendDataToSheet
[2021-08-06 23:06:01][4376] Processing: Maatwebsite\Excel\Jobs\CloseSheet
[2021-08-06 23:06:01][4376] Processed:  Maatwebsite\Excel\Jobs\CloseSheet
[2021-08-06 23:06:01][4377] Processing: Maatwebsite\Excel\Jobs\StoreQueuedExport
[2021-08-06 23:06:01][4377] Processed:  Maatwebsite\Excel\Jobs\StoreQueuedExport
[2021-08-06 23:06:01][4374] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:01][4378] Processing: Maatwebsite\Excel\Jobs\QueueExport
[2021-08-06 23:06:01][4378] Processed:  Maatwebsite\Excel\Jobs\QueueExport
[2021-08-06 23:06:02][4379] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:02][4379] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:02][4380] Processing: Maatwebsite\Excel\Jobs\AppendDataToSheet
[2021-08-06 23:06:02][4380] Processed:  Maatwebsite\Excel\Jobs\AppendDataToSheet
[2021-08-06 23:06:02][4381] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:03][4381] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:03][4382] Processing: Maatwebsite\Excel\Jobs\CloseSheet
[2021-08-06 23:06:03][4382] Processed:  Maatwebsite\Excel\Jobs\CloseSheet
[2021-08-06 23:06:03][4384] Processing: Maatwebsite\Excel\Jobs\StoreQueuedExport
[2021-08-06 23:06:04][4384] Processed:  Maatwebsite\Excel\Jobs\StoreQueuedExport
[2021-08-06 23:06:03][4383] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:04][4385] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:04][4383] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:04][4385] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:04][4386] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:05][4386] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:05][4387] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:05][4387] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:05][4388] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:06][4388] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:06][4389] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:07][4389] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:07][4390] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:07][4391] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:07][4390] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:07][4391] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:08][4392] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:08][4392] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:08][4393] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:08][4393] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:09][4394] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:09][4394] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:09][4395] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:09][4395] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:10][4396] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:10][4396] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:10][4397] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:11][4397] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:11][4398] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:11][4398] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:11][4399] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:12][4399] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:12][4400] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:12][4400] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:12][4401] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:13][4401] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:13][4402] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:13][4402] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:13][4403] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:14][4403] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:14][4404] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:14][4404] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:14][4405] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:15][4405] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:15][4406] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:15][4406] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:15][4407] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:16][4407] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:16][4408] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:17][4408] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:17][4409] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:17][4409] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:17][4410] Processing: Maatwebsite\Excel\Jobs\ReadChunk
[2021-08-06 23:06:18][4410] Processed:  Maatwebsite\Excel\Jobs\ReadChunk
.
.
.

[BehroozBvk]

Does anyone have an opinion?

[patrickbrouwers]

Try increasing the chunk size

[BehroozBvk]

thanks solved.

[SaaberDev]

@BehroozBvk hi, I used the same code as yours but it creates multiple files for each chunks ...

Is there any way to write in one file but in this manner ... for example: first chunk will create the file with first chunk result and when next failure chunk comes it will not override the file but write inside it.