Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 42 additions & 12 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ jobs:
install-firefox: false
install-geckodriver: false
install-chrome: true
chrome-version: "132.0.6834.110"
chrome-version: "142.0.7444.175"
- attach_workspace:
at: ~/
- run:
Expand Down Expand Up @@ -83,7 +83,7 @@ jobs:
install-firefox: false
install-geckodriver: false
install-chrome: true
chrome-version: "132.0.6834.110"
chrome-version: "142.0.7444.175"
- attach_workspace:
at: ~/
- run:
Expand All @@ -105,7 +105,7 @@ jobs:
install-firefox: false
install-geckodriver: false
install-chrome: true
chrome-version: "132.0.6834.110"
chrome-version: "142.0.7444.175"
- attach_workspace:
at: ~/
- run:
Expand All @@ -127,7 +127,7 @@ jobs:
install-firefox: false
install-geckodriver: false
install-chrome: true
chrome-version: "132.0.6834.110"
chrome-version: "142.0.7444.175"
- attach_workspace:
at: ~/
- run:
Expand Down Expand Up @@ -168,7 +168,7 @@ jobs:
install-firefox: false
install-geckodriver: false
install-chrome: true
chrome-version: "132.0.6834.110"
chrome-version: "142.0.7444.175"
- attach_workspace:
at: ~/
- run:
Expand All @@ -189,7 +189,7 @@ jobs:
install-firefox: false
install-geckodriver: false
install-chrome: true
chrome-version: "132.0.6834.110"
chrome-version: "142.0.7444.175"
- attach_workspace:
at: ~/
- run:
Expand Down Expand Up @@ -217,9 +217,15 @@ jobs:
make-baselines-virtual-webgl:
parallelism: 8
docker:
- image: cimg/python:3.12.11
- image: cimg/python:3.12.11-browsers
working_directory: ~/plotly.js
steps:
- run: sudo apt-get update
- browser-tools/install-browser-tools:
install-firefox: false
install-geckodriver: false
install-chrome: true
chrome-version: "142.0.7444.175"
- attach_workspace:
at: ~/
- run: sudo apt-get update
Expand All @@ -236,9 +242,15 @@ jobs:

make-baselines-mathjax3:
docker:
- image: cimg/python:3.12.11
- image: cimg/python:3.12.11-browsers
working_directory: ~/plotly.js
steps:
- run: sudo apt-get update
- browser-tools/install-browser-tools:
install-firefox: false
install-geckodriver: false
install-chrome: true
chrome-version: "142.0.7444.175"
- attach_workspace:
at: ~/
- run: sudo apt-get update
Expand All @@ -256,9 +268,15 @@ jobs:
make-baselines:
parallelism: 12
docker:
- image: cimg/python:3.12.11
- image: cimg/python:3.12.11-browsers
working_directory: ~/plotly.js
steps:
- run: sudo apt-get update
- browser-tools/install-browser-tools:
install-firefox: false
install-geckodriver: false
install-chrome: true
chrome-version: "142.0.7444.175"
- attach_workspace:
at: ~/
- run: sudo apt-get update
Expand All @@ -276,9 +294,15 @@ jobs:
make-baselines-b64:
parallelism: 12
docker:
- image: cimg/python:3.12.11
- image: cimg/python:3.12.11-browsers
working_directory: ~/plotly.js
steps:
- run: sudo apt-get update
- browser-tools/install-browser-tools:
install-firefox: false
install-geckodriver: false
install-chrome: true
chrome-version: "142.0.7444.175"
- attach_workspace:
at: ~/
- run: sudo apt-get update
Expand Down Expand Up @@ -355,9 +379,15 @@ jobs:

make-exports:
docker:
- image: cimg/python:3.12.11
- image: cimg/python:3.12.11-browsers
working_directory: ~/plotly.js
steps:
- run: sudo apt-get update
- browser-tools/install-browser-tools:
install-firefox: false
install-geckodriver: false
install-chrome: true
chrome-version: "142.0.7444.175"
- attach_workspace:
at: ~/
- run: sudo apt-get update
Expand All @@ -371,7 +401,7 @@ jobs:
sudo apt-get install poppler-utils
- run:
name: Create svg, jpg, jpeg, webp, pdf and eps files
command: sudo python3 test/image/make_exports.py
command: uv run test/image/make_exports.py
- persist_to_workspace:
root: ~/
paths:
Expand Down
19 changes: 12 additions & 7 deletions .circleci/env_image.sh
Original file line number Diff line number Diff line change
@@ -1,21 +1,26 @@
#!/bin/sh
#!/bin/bash
set -e
# install required fonts
sudo apt-get install fonts-liberation2 fonts-open-sans fonts-noto-cjk fonts-noto-color-emoji

# install pip
sudo curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
sudo python3 get-pip.py
curl -LsSf https://astral.sh/uv/install.sh | sh
uv venv
source .venv/bin/activate

# install additional fonts
sudo python3 -m pip install requests
sudo python3 .circleci/download_google_fonts.py
uv pip install requests
python .circleci/download_google_fonts.py
sudo cp -r .circleci/fonts/ /usr/share/
sudo apt install fontconfig
sudo fc-cache -f

# install kaleido & plotly
sudo python3 -m pip install kaleido==0.2.1 plotly==6.2.0 --progress-bar off
uv pip install "plotly==6.5" "kaleido==1.2" --no-progress

# install numpy i.e. to convert arrays to typed arrays
sudo python3 -m pip install numpy==1.24.2
uv pip install "numpy==2.3" --no-progress

# verify version of python and versions of installed python packages
python --version
uv pip freeze
8 changes: 4 additions & 4 deletions .circleci/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -97,24 +97,24 @@ case $1 in
find $ROOT/test/image/mocks/gl* -type f -printf "%f\n"; \
find $ROOT/test/image/mocks/map* -type f -printf "%f\n"; \
} | sed 's/\.json$//1' | circleci tests split)
sudo python3 test/image/make_baseline.py virtual-webgl $SUITE || EXIT_STATE=$?
uv run test/image/make_baseline.py virtual-webgl $SUITE || EXIT_STATE=$?
exit $EXIT_STATE
;;

make-baselines-mathjax3)
sudo python3 test/image/make_baseline.py mathjax3 legend_mathjax_title_and_items mathjax parcats_grid_subplots table_latex_multitrace_scatter table_plain_birds table_wrapped_birds ternary-mathjax ternary-mathjax-title-place-subtitle || EXIT_STATE=$?
uv run test/image/make_baseline.py mathjax3 legend_mathjax_title_and_items mathjax parcats_grid_subplots table_latex_multitrace_scatter table_plain_birds table_wrapped_birds ternary-mathjax ternary-mathjax-title-place-subtitle || EXIT_STATE=$?
exit $EXIT_STATE
;;

make-baselines-b64)
SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
sudo python3 test/image/make_baseline.py b64 $SUITE || EXIT_STATE=$?
uv run test/image/make_baseline.py b64 $SUITE || EXIT_STATE=$?
exit $EXIT_STATE
;;

make-baselines)
SUITE=$(find $ROOT/test/image/mocks/ -type f -printf "%f\n" | sed 's/\.json$//1' | circleci tests split)
sudo python3 test/image/make_baseline.py $SUITE || EXIT_STATE=$?
uv run test/image/make_baseline.py $SUITE || EXIT_STATE=$?
exit $EXIT_STATE
;;

Expand Down
139 changes: 82 additions & 57 deletions test/image/make_baseline.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
import asyncio
import json
import os
import sys
import json

import kaleido
import plotly.io as pio

from convert_b64 import arraysToB64


args = []
if len(sys.argv) == 2 :
args = sys.argv[1].split()
Expand All @@ -19,7 +24,7 @@
dirIn = os.path.join(root, 'test', 'image', 'mocks')
dirOut = os.path.join(root, 'build', 'test_images')

# N.B. equal is the falg to write to baselines not test_images
# N.B. equal is the flag to write to baselines not test_images

if '=' in args :
args = args[args.index('=') + 1:]
Expand All @@ -31,11 +36,12 @@
print('output to', dirOut)

mathjax_version = 2
mathjax = None
if 'mathjax3' in sys.argv or 'mathjax3=' in sys.argv :
# until https://github.com/plotly/Kaleido/issues/124 is addressed
# we are uanble to use local mathjax v3 installed in node_modules
# for now let's download it from the internet:
pio.kaleido.scope.mathjax = 'https://cdn.jsdelivr.net/npm/[email protected]/es5/tex-svg.js'
mathjax = 'https://cdn.jsdelivr.net/npm/[email protected]/es5/tex-svg.js'
mathjax_version = 3
print('Kaleido using MathJax v3')

Expand All @@ -52,8 +58,8 @@

plotlyjs = plotlyjs_with_virtual_webgl

pio.kaleido.scope.plotlyjs = plotlyjs
pio.kaleido.scope.topojson = "file://" + os.path.join(root, 'topojson', 'dist')
pio.defaults.plotlyjs = plotlyjs
pio.defaults.topojson = "file://" + os.path.join(root, 'topojson', 'dist')
pio.templates.default = 'none'

ALL_MOCKS = [os.path.splitext(a)[0] for a in os.listdir(dirIn) if a.endswith('.json')]
Expand All @@ -79,55 +85,74 @@
sys.exit(1)

failed = []
for name in allNames :
outName = name
if mathjax_version == 3 :
outName = 'mathjax3___' + name

print(outName)

created = False

MAX_RETRY = 2 # 1 means retry once
for attempt in range(0, MAX_RETRY + 1) :
with open(os.path.join(dirIn, name + '.json'), 'r') as _in :
fig = json.load(_in)

width = 700
height = 500
if 'layout' in fig :
layout = fig['layout']
if 'autosize' not in layout or layout['autosize'] != True :
if 'width' in layout :
width = layout['width']
if 'height' in layout :
height = layout['height']

if 'b64' in sys.argv or 'b64=' in sys.argv or 'b64-json' in sys.argv :
newFig = dict()
arraysToB64(fig, newFig)
fig = newFig
if 'b64-json' in sys.argv and attempt == 0 : print(json.dumps(fig, indent = 2))

try :
pio.write_image(
fig=fig,
file=os.path.join(dirOut, outName + '.png'),
width=width,
height=height,
validate=False
)
created = True
except Exception as e :
print(e)
if attempt < MAX_RETRY :
print('retry', attempt + 1, '/', MAX_RETRY)
else :
failed.append(outName)

if(created) : break

if len(failed) > 0 :
print('Failed at :')
print(failed)
sys.exit(1)

async def make_baselines_async():

kopts = dict(
plotlyjs=plotlyjs,
)
if mathjax is not None:
kopts['mathjax'] = mathjax

async with kaleido.Kaleido(n=1, **kopts) as k:
for name in allNames:
outName = name
if mathjax_version == 3:
outName = 'mathjax3___' + name

print(outName)

created = False

MAX_RETRY = 2 # 1 means retry once
for attempt in range(0, MAX_RETRY + 1) :
with open(os.path.join(dirIn, name + '.json'), 'r') as _in :
fig = json.load(_in)

width = 700
height = 500
if 'layout' in fig :
layout = fig['layout']
if 'autosize' not in layout or layout['autosize'] != True :
if 'width' in layout :
width = layout['width']
if 'height' in layout :
height = layout['height']

if 'b64' in sys.argv or 'b64=' in sys.argv or 'b64-json' in sys.argv :
newFig = dict()
arraysToB64(fig, newFig)
fig = newFig
if 'b64-json' in sys.argv and attempt == 0 : print(json.dumps(fig, indent = 2))

try:
bytes = await k.calc_fig(
fig,
path=None,
opts=dict(
format="png",
width=width,
height=height,
),
)
filename = os.path.join(dirOut, outName + '.png')
with open(filename, "wb") as f:
f.write(bytes)
created = True
except Exception as e:
print(e)
if attempt < MAX_RETRY :
print('retry', attempt + 1, '/', MAX_RETRY)
else :
failed.append(outName)

if(created): break

if len(failed) > 0 :
print('Failed at :')
print(failed)
sys.exit(1)


if __name__ == "__main__":
asyncio.run(make_baselines_async())
Loading