Why queued imports don't reach end of file ?

[atefrihane]

I've a CSV file with 5794 records
I'm trying to import it to database..
I set the chunk size to 2000 , it doesn't process all the file and hangs at 3000 insertions with no errors.
I tried to decrease it even to 25 and 10 and always doesn't reach the end..
What's causing this ?

<?php

namespace App\Imports;

use App\Modules\Metric\Models\Metric;
use App\Modules\Moment\Models\Moment;
use App\Modules\User\Models\User;
use Carbon\Carbon;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Support\Collection;
use Maatwebsite\Excel\Concerns\ToCollection;
use Maatwebsite\Excel\Concerns\WithChunkReading;
use Maatwebsite\Excel\Concerns\WithStartRow;
use Throwable;

class MomentsImport implements ToCollection, WithStartRow, WithChunkReading, ShouldQueue
{

    public function startRow(): int
    {
        return 1;
    }
    public function collection(Collection $collection)
    {

        $insertedIds = [];

        try {
            foreach ($collection as $row) {

                $chunkOfUsers = explode("+", $row[2]);

                $moment = $this->createMoment($row);

                if ($moment) {
                    foreach ($chunkOfUsers as $chunkUser) {

                        $checkUser = $this->checkUser($chunkUser);
                        if (!$checkUser) {

                            $insertedUser = $this->insertUser($chunkUser, $row, $moment);

                            $insertedIds[] = $insertedUser['id'];

                        }
                        $checkUser ? $insertedIds[] = $checkUser->id : "";

                    }
                    count($insertedIds) > 0 ? $this->insertCypherMoments($insertedIds, $row) : "";

                    $insertedIds = [];
                }

            }

        } catch (Throwable $e) {
            report($e);

            return false;

        }

    }

    public function runCypher($query)
    {
        $bolt = new \Bolt\Bolt(new \Bolt\connection\StreamSocket());
        // Set Bolt protocol version if needed
        $bolt->setProtocolVersions(config('memgraph.protocol_version'));
        // Connect to database
        $bolt->init('MyClient/1.0', config('memgraph.username'), config('memgraph.password'));
        $bolt->run($query);
        $result = $bolt->pull();
        return $result;
    }

    public function chunkSize(): int
    {
        return 2000;
    }

    public function checkUser($fullName)
    {
        $fullNameArray = explode(" ", $fullName);
        $checkUser = User::where('first_name', $fullNameArray[0] ? $fullNameArray[0] : $fullNameArray[1])
            ->where('last_name', $fullNameArray[1] ? $fullNameArray[1] : $fullNameArray[0])
            ->first();

        return $checkUser ? $checkUser : false;
    }

    public function insertUser($fullName, $row, $moment)
    {

        $faker = \Faker\Factory::create();

        try {

            // Validate the value...
            $fullNameArray = explode(" ", $fullName);
            //remove empty  key values
            $fullNameArray = array_values(array_filter(array_map('trim', $fullNameArray), 'strlen'));

            $newUser = User::Create([
                'first_name' => isset($fullNameArray[0]) ? $fullNameArray[0] : $fullNameArray[1],
                'last_name' => isset($fullNameArray[1]) ? $fullNameArray[1] : $fullNameArray[0],
                'email' => $faker->unique()->email,
                'password' => bcrypt(config('constants.default_password')),

            ]);

            $moment->users()->attach($newUser->id);

            $query = "CREATE (u:User {id_user_db:'$newUser->id', email:'$newUser->email',first_name:'$newUser->first_name',last_name:'$newUser->last_name',user_type:'legacy' });";

            $this->runCypher($query);
            return $newUser->toArray();
        } catch (Throwable $e) {
            dump($query);
            dd($e);
            report($e);

            return false;
        }

    }

    public function createMoment($row)
    {

        $type = $row[1];

        try {

            // $query = "Match (a)-[r]->(b)
            //     WHERE type(r)='$row[1]' AND r.description='$row[5]' AND r.created_at='$row[0]'
            //     Return a,b,r";

            // $result = $this->runCypher($query);

            // if (count($result) > 1) {
            //     return false;
            // }

            $metric = Metric::firstOrCreate([
                'name' => $type,
            ]);
            $moment = Moment::create([
                'edge_id' => $row[4],
                'tool' => $row[7],
                'description' => $row[5],
                'metric_id' => $metric->id,

                'created_at' => Carbon::parse(strtotime($row[0])),
            ]);

            return $moment;

        } catch (Throwable $e) {

            report($e);

            return false;
        }

    }

    public function insertCypherMoments($insertedIds, $row)
    {

        $type = $row[1];

        try {
            foreach ($insertedIds as $insertedId) {
                $localIds = $insertedIds;

                if (count($localIds) > 1) {
                    $index = array_search($insertedId, $localIds);

                    if ($index !== false) {
                        unset($localIds[$index]); // $arr = ['b', 'c']
                    }
                }

                foreach ($localIds as $localId) {

                    $query = "
                    MATCH (a:User),(b:User)
                    WHERE a.id_user_db= '$insertedId' AND b.id_user_db = '$localId'
                    MATCH (a)-[r:$type]-(b)
                    RETURN type(r)";

                    $result = $this->runCypher($query);

                }
            }

        } catch (Throwable $e) {
            dump($query);
            dd($e);

            report($e);

            return false;
        }
    }

}

[patrickbrouwers]

Probably a memory exhaustion, try to disable your own logic and see if it runs to the end then. If so, you'll have to optimize your own logic. (Try enabling things one by one and see which parts uses a lot of memory)