Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ on:
branches:
- "19.0"
- "19.0-ocabot-*"
- "19.0-web_coverage"

jobs:
unreleased-deps:
Expand Down
1 change: 1 addition & 0 deletions web_coverage/README.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
dummy
4 changes: 4 additions & 0 deletions web_coverage/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
from . import parse_asset_bundle
from . import patch_browser
from . import patch_test_loader
from . import process_js_coverage
7 changes: 7 additions & 0 deletions web_coverage/__manifest__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "JS coverage",
"version": "19.0.1.0.0",
"license": "AGPL-3",
"author": "Hunki Enterprises BV, Odoo Community Association (OCA)",
"website": "https://github.com/OCA/web",
}
82 changes: 82 additions & 0 deletions web_coverage/parse_asset_bundle.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
import json

B64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
B64_MAP = {char: val for val, char in enumerate(B64_ALPHABET)}
CONTINUE = 0b100000
CONTINUE_COMPLEMENT = 0b011111
SIGN = 0b000001


def b64vlq_decode(chars):
"""
Decode a base64 vlq string into a tuple of integers
"""
vals = []
current_val, shift = 0, 0
for c in chars:
bits = B64_MAP[c]
current_val = (bits & CONTINUE_COMPLEMENT) << shift | current_val
shift += 5
if not bits & CONTINUE:
sign = current_val & SIGN and -1 or 1
vals.append((current_val >> 1) * sign)
current_val, shift = 0, 0
return tuple(vals)


def parse_asset_bundle(js_attachment, sourcemap_attachment):
"""
Return list of source files in bundle, and a list mapping indices in js_attachment
to asource file and a line therein
"""
sourcemap = json.loads(sourcemap_attachment.raw[5:])
sourcemap["mappings"] = [
tuple(b64vlq_decode(segment) for segment in group.split(","))
for group in sourcemap["mappings"].split(";")
]

sources = [{"name": file_name} for file_name in sourcemap["sources"]]
sources_index = 0
# TODO: indices in coverage don't map exactly those in
# js_attachment.raw.decode('utf8') - for every
# 4-byte emoji in the code, we get offset+1
# the below fixes this, but why?
gen_content = "".join(
map(
lambda c: (c + c) if len(c.encode("utf8")) == 4 else c,
js_attachment.raw.decode("utf8"),
)
)
index2file_line = [None] * len(gen_content)
gen_index = 0
org_line = 0
org_column = 0

for line_mappings, line in zip(
# strict=False because bundle attachment has appended code for template which
# is not mapped
sourcemap["mappings"],
gen_content.splitlines(keepends=True),
strict=False,
):
gen_column = 0
for mapping in line_mappings:
if not mapping:
continue

gen_column_delta, sources_index_delta, org_line_delta, org_column_delta = (
mapping
)

gen_column += gen_column_delta
sources_index += sources_index_delta
org_line += org_line_delta
org_column += org_column_delta

next_gen_index = gen_index + len(line)
index2file_line[gen_index:next_gen_index] = [
(sources[sources_index], org_line)
] * (next_gen_index - gen_index)
gen_index = next_gen_index

return sources, index2file_line
58 changes: 58 additions & 0 deletions web_coverage/patch_browser.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import json
from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit

from odoo.tools import mute_logger

with mute_logger("odoo.tests.common"):
from odoo.tests.common import ChromeBrowser, save_test_file


_wait_code_ok_org = ChromeBrowser._wait_code_ok
navigate_to_org = ChromeBrowser.navigate_to


def navigate_to(self, url, *args, **kwargs):
self._websocket_request("Profiler.enable")
self._websocket_request("Profiler.startPreciseCoverage", params={"detailed": True})
# we need to run unminified JS files to be able to map coverage to lines in files
parsed_url = urlsplit(url)
new_url = urlunsplit(
parsed_url[:3]
+ (urlencode(dict(parse_qs(parsed_url.query), debug="assets"), doseq=True),)
+ parsed_url[4:]
)
return navigate_to_org(self, new_url, *args, **kwargs)


def _wait_code_ok(self, *args, **kwargs):
result = _wait_code_ok_org(self, *args, **kwargs)

coverage = self._websocket_request("Profiler.takePreciseCoverage")
if not coverage or not coverage["result"]:
self._logger.warning("No coverage collected")
return result
self._websocket_request("Debugger.enable")
for script_coverage in coverage["result"]:
if not script_coverage["url"] or "/debug/" not in script_coverage["url"]:
# we don't care for files not in some debug asset
self._logger.debug(
"ignoring script %s: %s",
script_coverage["scriptId"],
script_coverage["url"],
)
continue
save_test_file(
self.test_case._testMethodName,
json.dumps(script_coverage).encode("utf8"),
prefix=f"{script_coverage['scriptId']}_",
extension="json",
document_type="JS coverage",
logger=self._logger,
loglevel=self.chrome_log_level,
directory="web_coverage",
)
return result


ChromeBrowser._wait_code_ok = _wait_code_ok
ChromeBrowser.navigate_to = navigate_to
44 changes: 44 additions & 0 deletions web_coverage/patch_test_loader.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import threading

from odoo import api
from odoo.orm.registry import Registry
from odoo.tools import mute_logger

with mute_logger("odoo.tests.common"):
from odoo.tests import loader
from . import process_js_coverage

run_suite_org = loader.run_suite


def run_suite(suite, *args, **kwargs):
if suite.has_http_case():
dbname = threading.current_thread().dbname
registry = Registry.registries[dbname]
with registry.cursor() as cr:
# pregenerate debug asset bundles because we patch debug=assets into all
# urls in order to avoid minification to be able to match covered code
# with code in the filesystem
env = api.Environment(cr, api.SUPERUSER_ID, {})
IrQweb = env["ir.qweb"]
js_bundles, css_bundles = IrQweb._get_bundles_to_pregenarate()
for bundle in sorted(js_bundles):
IrQweb._get_asset_bundle(
bundle, css=False, js=True, debug_assets=True
).js()
for bundle in sorted(css_bundles):
IrQweb._get_asset_bundle(
bundle, css=True, js=False, debug_assets=True
).css()

result = run_suite_org(suite, *args, **kwargs)
if suite.has_http_case():
dbname = threading.current_thread().dbname
registry = Registry.registries[dbname]
with registry.cursor() as cr:
env = api.Environment(cr, api.SUPERUSER_ID, {})
process_js_coverage.process(env, loader._logger)
return result


loader.run_suite = run_suite
86 changes: 86 additions & 0 deletions web_coverage/process_js_coverage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
import json
import pathlib
from urllib.parse import urlparse

from odoo.tools import config, file_path

from .parse_asset_bundle import parse_asset_bundle


def process(env, _logger):
coverage_path = pathlib.Path(config["screenshots"]) / env.cr.dbname / "web_coverage"
if not coverage_path.is_dir():
return
# TODO: find all files in os.env['WEB_COVERAGE_INCLUDE'],
# initialize missed with all lines
all_files = {}
for coverage_file in coverage_path.iterdir():
coverage = json.loads(coverage_file.read_text())
url = urlparse(coverage["url"])
split_name = url.path.split("/")[-1].split(".")
while split_name and split_name[-1] in ("min", "js"):
split_name.pop(-1)
bundle_name = ".".join(split_name)
asset_bundle = env["ir.qweb"]._get_asset_bundle(
bundle_name, css=False, js=True, debug_assets=True
)
js_attachment = asset_bundle.get_attachments("js")
sourcemap_attachment = asset_bundle.get_attachments("js.map")
if not sourcemap_attachment:
_logger.debug(
"sourcemap file for bundle %s not found - ignoring", bundle_name
)
continue

sources, index2file_line = parse_asset_bundle(
js_attachment, sourcemap_attachment
)

for file_coverage in sources:
file_name_relative = file_coverage["name"][1:]
file_name_fs = file_path(file_name_relative)
if not file_name_fs:
_logger.debug("bundle file %s not found - ignoring", file_name_relative)
file_coverage["line_count"] = 0
file_coverage["missed"] = set()
continue
file_coverage["line_count"] = len(
pathlib.Path(file_name_fs).read_text().splitlines()
)
file_coverage["missed"] = set()

for function in coverage["functions"]:
for coverage_range in function["ranges"]:
if not coverage_range["count"]:
start = coverage_range["startOffset"]
end = coverage_range["endOffset"]
file_coverage = index2file_line[start][0]
assert file_coverage == index2file_line[end][0]
file_coverage["missed"] |= set(
range(index2file_line[start][1], index2file_line[end][1] + 1)
)

for file_coverage in sources:
if not file_coverage["line_count"]:
continue
file_name_relative = file_coverage["name"][1:]
# TODO: fix this
if not pathlib.Path(file_name_relative).exists():
continue
if file_name_relative in all_files:
all_files[file_name_relative]["missed"] &= file_coverage["missed"]
else:
all_files[file_name_relative] = file_coverage

with pathlib.Path("lcov.info").open("w+") as lcov_file:
lcov_file.write("TN:\n")
for file_name, file_coverage in all_files.items():
lcov_file.write(f"SF:{file_name}\n")
lcov_file.write(f"LF:{file_coverage['line_count']}\n")
lcov_file.write(
f"LH:{file_coverage['line_count'] - len(file_coverage['missed'])}\n"
)
for line in range(file_coverage["line_count"]):
hit = int(line not in file_coverage["missed"])
lcov_file.write(f"DA:{line + 1},{hit}\n")
lcov_file.write("end_of_record\n")
3 changes: 3 additions & 0 deletions web_coverage/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[build-system]
requires = ["whool"]
build-backend = "whool.buildapi"