{ "computedAt": "2026-11-12T10:48:42.108Z", "lockHash": "434aac813fc621eea340922736df2d3353045b6bf88e448d086063266d5191bf", "weights": { "1": 20, "2": 10, "3": 15, "4": 10, "5": 15, "6": 10, "7": 10, "8": 10 }, "dimensionNames": { "1": "Case volume and specificity", "2": "Framework breadth", "3": "Framework depth", "4": "Migration evidence", "5": "Outcome verifiability", "6": "Third-party review signal", "7": "Public engineering output", "8": "Transparency" }, "vendors": [ { "slug": "deviqa", "name": "DeviQA", "rank": 1, "ranked": true, "overall": 83.5, "dimensions": { "1": { "score": 10, "measured": "28.4 weighted cases", "rationale": "52 verified cases: A 15, B 36, C 1, weighted 1.0, 0.6 and 0.3; 27 historical cases at half weight.", "ids": [ "deviqa-1", "deviqa-2", "deviqa-3", "deviqa-4", "deviqa-5", "deviqa-6", "deviqa-7", "deviqa-8", "deviqa-9", "deviqa-10", "deviqa-11", "deviqa-12", "deviqa-13", "deviqa-14", "deviqa-15", "deviqa-16", "deviqa-17", "deviqa-18", "deviqa-19", "deviqa-20", "deviqa-21", "deviqa-22", "deviqa-23", "deviqa-24", "deviqa-25", "deviqa-26", "deviqa-27", "deviqa-28", "deviqa-29", "deviqa-30", "deviqa-31", "deviqa-32", "deviqa-33", "deviqa-34", "deviqa-35", "deviqa-36", "deviqa-37", "deviqa-38", "deviqa-39", "deviqa-40", "deviqa-41", "deviqa-42", "deviqa-43", "deviqa-44", "deviqa-45", "deviqa-46", "deviqa-47", "deviqa-48", "deviqa-49", "deviqa-50", "deviqa-51", "deviqa-52" ] }, "2": { "score": 10, "measured": "12 frameworks", "rationale": "Frameworks with at least one A or B case: Playwright, Postman and Newman, Cypress, Selenium WebDriver, Apache JMeter, Grafana k6, Appium, WebdriverIO, Cucumber, SpecFlow, Behave, REST Assured, Locust, Robot Framework.", "ids": [ "deviqa-1", "deviqa-2", "deviqa-3", "deviqa-7", "deviqa-8", "deviqa-21", "deviqa-33", "deviqa-46", "deviqa-52", "deviqa-5", "deviqa-6", "deviqa-10", "deviqa-11", "deviqa-12", "deviqa-13", "deviqa-16", "deviqa-17", "deviqa-18", "deviqa-22", "deviqa-23", "deviqa-24", "deviqa-25", "deviqa-26", "deviqa-28", "deviqa-29", "deviqa-31", "deviqa-32", "deviqa-34", "deviqa-35", "deviqa-36", "deviqa-38", "deviqa-39", "deviqa-40", "deviqa-41", "deviqa-44", "deviqa-45", "deviqa-47", "deviqa-48", "deviqa-49", "deviqa-50", "deviqa-4", "deviqa-20", "deviqa-42", "deviqa-9", "deviqa-19", "deviqa-43", "deviqa-51", "deviqa-15", "deviqa-30", "deviqa-37", "deviqa-27" ] }, "3": { "score": 10, "measured": "36 cases on Postman and Newman", "rationale": "Most cases on a single framework: Postman and Newman.", "ids": [ "deviqa-1", "deviqa-3", "deviqa-5", "deviqa-6", "deviqa-8", "deviqa-10", "deviqa-11", "deviqa-12", "deviqa-13", "deviqa-16", "deviqa-17", "deviqa-18", "deviqa-22", "deviqa-23", "deviqa-24", "deviqa-25", "deviqa-26", "deviqa-28", "deviqa-29", "deviqa-31", "deviqa-32", "deviqa-34", "deviqa-35", "deviqa-36", "deviqa-38", "deviqa-39", "deviqa-40", "deviqa-41", "deviqa-44", "deviqa-45", "deviqa-46", "deviqa-47", "deviqa-48", "deviqa-49", "deviqa-50", "deviqa-52" ] }, "4": { "score": 7, "measured": "2 migration cases with before and after", "rationale": "2 cases tagged as migration; 2 state a before and an after value.", "ids": [ "deviqa-2", "deviqa-4" ] }, "5": { "score": 3, "measured": "32 of 227 metrics", "rationale": "32 of 227 published metrics state both a baseline and a method.", "ids": [ "deviqa-2", "deviqa-3", "deviqa-9", "deviqa-11", "deviqa-15", "deviqa-16", "deviqa-17", "deviqa-20", "deviqa-29", "deviqa-33", "deviqa-37", "deviqa-42", "deviqa-43", "deviqa-44", "deviqa-51", "deviqa-52" ] }, "6": { "score": 9, "measured": "6 reviews naming a framework", "rationale": "Verified reviews within 60 months whose text names a listed framework; newest within 24 months adds 1.", "ids": [ "ev-deviqa-006", "ev-deviqa-007", "ev-deviqa-008", "ev-deviqa-009", "ev-deviqa-010", "ev-deviqa-011" ] }, "7": { "score": 8, "measured": "12 items", "rationale": "Dated posts, talks or repositories by staff about a listed framework within 36 months.", "ids": [ "ev-deviqa-013", "ev-deviqa-014", "ev-deviqa-015", "ev-deviqa-016", "ev-deviqa-017", "ev-deviqa-018", "ev-deviqa-019", "ev-deviqa-020", "ev-deviqa-021", "ev-deviqa-022", "ev-deviqa-023", "ev-deviqa-024" ] }, "8": { "score": 10, "measured": "4 of 4 items", "rationale": "Disclosed: public rate, named leadership with profiles, QA process page, delivery locations.", "ids": [ "ev-deviqa-001", "ev-deviqa-004", "ev-deviqa-005", "ev-deviqa-003" ] } }, "grades": { "A": 15, "B": 36, "C": 1 }, "frameworksAB": [ "postman", "appium", "jmeter", "cucumber-bdd", "webdriverio", "playwright", "selenium", "cypress", "rest-assured", "k6", "robot-framework", "locust" ] }, { "slug": "codoid", "name": "Codoid", "rank": 2, "ranked": true, "overall": 65, "dimensions": { "1": { "score": 10, "measured": "14.6 weighted cases", "rationale": "40 verified cases: A 0, B 10, C 30, weighted 1.0, 0.6 and 0.3; 2 historical cases at half weight.", "ids": [ "codoid-1", "codoid-2", "codoid-3", "codoid-4", "codoid-5", "codoid-6", "codoid-7", "codoid-8", "codoid-9", "codoid-10", "codoid-11", "codoid-12", "codoid-13", "codoid-14", "codoid-15", "codoid-16", "codoid-17", "codoid-18", "codoid-19", "codoid-20", "codoid-21", "codoid-22", "codoid-23", "codoid-24", "codoid-25", "codoid-26", "codoid-27", "codoid-28", "codoid-29", "codoid-30", "codoid-31", "codoid-32", "codoid-33", "codoid-34", "codoid-35", "codoid-36", "codoid-37", "codoid-38", "codoid-39", "codoid-40" ] }, "2": { "score": 10, "measured": "8 frameworks", "rationale": "Frameworks with at least one A or B case: Cypress, Appium, WebdriverIO, REST Assured, Apache JMeter, Selenium WebDriver, Cucumber, SpecFlow, Behave, Playwright.", "ids": [ "codoid-1", "codoid-34", "codoid-6", "codoid-4", "codoid-32", "codoid-2", "codoid-3", "codoid-5", "codoid-10", "codoid-19" ] }, "3": { "score": 10, "measured": "19 cases on Selenium WebDriver", "rationale": "Most cases on a single framework: Selenium WebDriver.", "ids": [ "codoid-2", "codoid-3", "codoid-5", "codoid-9", "codoid-10", "codoid-11", "codoid-13", "codoid-14", "codoid-15", "codoid-19", "codoid-20", "codoid-21", "codoid-22", "codoid-23", "codoid-24", "codoid-27", "codoid-30", "codoid-35", "codoid-39" ] }, "4": { "score": 0, "measured": "0 migration cases with before and after", "rationale": "0 cases tagged as migration; 0 state a before and an after value.", "ids": [] }, "5": { "score": 1, "measured": "1 of 46 metrics", "rationale": "1 of 46 published metrics state both a baseline and a method.", "ids": [ "codoid-10" ] }, "6": { "score": 3, "measured": "1 review naming a framework", "rationale": "Verified reviews within 60 months whose text names a listed framework.", "ids": [ "ev-codoid-005" ] }, "7": { "score": 8, "measured": "12 items", "rationale": "Dated posts, talks or repositories by staff about a listed framework within 36 months.", "ids": [ "ev-codoid-010", "ev-codoid-011", "ev-codoid-012", "ev-codoid-013", "ev-codoid-014", "ev-codoid-015", "ev-codoid-016", "ev-codoid-017", "ev-codoid-018", "ev-codoid-019", "ev-codoid-020", "ev-codoid-021" ] }, "8": { "score": 7.5, "measured": "3 of 4 items", "rationale": "Disclosed: public rate, QA process page, delivery locations. Not found: named leadership with profiles.", "ids": [ "ev-codoid-004", "ev-codoid-008", "ev-codoid-002" ] } }, "grades": { "A": 0, "B": 10, "C": 30 }, "frameworksAB": [ "cucumber-bdd", "selenium", "jmeter", "appium", "cypress", "playwright", "rest-assured", "webdriverio" ] }, { "slug": "a1qa", "name": "a1qa", "rank": 3, "ranked": true, "overall": 60, "dimensions": { "1": { "score": 10, "measured": "22.5 weighted cases", "rationale": "61 verified cases: A 0, B 14, C 47, weighted 1.0, 0.6 and 0.3.", "ids": [ "a1qa-1", "a1qa-2", "a1qa-3", "a1qa-4", "a1qa-5", "a1qa-6", "a1qa-7", "a1qa-8", "a1qa-9", "a1qa-10", "a1qa-11", "a1qa-12", "a1qa-13", "a1qa-14", "a1qa-15", "a1qa-16", "a1qa-17", "a1qa-18", "a1qa-19", "a1qa-20", "a1qa-21", "a1qa-22", "a1qa-23", "a1qa-24", "a1qa-25", "a1qa-26", "a1qa-27", "a1qa-28", "a1qa-29", "a1qa-30", "a1qa-31", "a1qa-32", "a1qa-33", "a1qa-34", "a1qa-35", "a1qa-36", "a1qa-37", "a1qa-38", "a1qa-39", "a1qa-40", "a1qa-41", "a1qa-42", "a1qa-44", "a1qa-60", "a1qa-61", "a1qa-62", "a1qa-63", "a1qa-64", "a1qa-65", "a1qa-66", "a1qa-67", "a1qa-68", "a1qa-69", "a1qa-70", "a1qa-71", "a1qa-72", "a1qa-73", "a1qa-74", "a1qa-75", "a1qa-76", "a1qa-77" ] }, "2": { "score": 10, "measured": "8 frameworks", "rationale": "Frameworks with at least one A or B case: Selenium WebDriver, REST Assured, Playwright, Cucumber, SpecFlow, Behave, Apache JMeter, Postman and Newman, Gatling, Detox.", "ids": [ "a1qa-1", "a1qa-4", "a1qa-5", "a1qa-23", "a1qa-29", "a1qa-63", "a1qa-2", "a1qa-3", "a1qa-9", "a1qa-67", "a1qa-7", "a1qa-11", "a1qa-25", "a1qa-30" ] }, "3": { "score": 10, "measured": "26 cases on Postman and Newman", "rationale": "Most cases on a single framework: Postman and Newman.", "ids": [ "a1qa-4", "a1qa-6", "a1qa-9", "a1qa-11", "a1qa-12", "a1qa-14", "a1qa-15", "a1qa-16", "a1qa-24", "a1qa-25", "a1qa-30", "a1qa-31", "a1qa-32", "a1qa-33", "a1qa-34", "a1qa-39", "a1qa-41", "a1qa-42", "a1qa-60", "a1qa-62", "a1qa-64", "a1qa-67", "a1qa-70", "a1qa-72", "a1qa-74", "a1qa-77" ] }, "4": { "score": 0, "measured": "0 migration cases with before and after", "rationale": "1 case tagged as migration; 0 state a before and an after value.", "ids": [] }, "5": { "score": 3, "measured": "14 of 98 metrics", "rationale": "14 of 98 published metrics state both a baseline and a method.", "ids": [ "a1qa-1", "a1qa-2", "a1qa-3", "a1qa-4", "a1qa-5", "a1qa-9", "a1qa-23", "a1qa-25", "a1qa-29", "a1qa-30", "a1qa-63" ] }, "6": { "score": 0, "measured": "0 reviews naming a framework", "rationale": "No verified review within 60 months names a listed framework.", "ids": [] }, "7": { "score": 3, "measured": "2 items", "rationale": "Dated posts, talks or repositories by staff about a listed framework within 36 months.", "ids": [ "ev-a1qa-005", "ev-a1qa-006" ] }, "8": { "score": 7.5, "measured": "3 of 4 items", "rationale": "Disclosed: public rate, QA process page, delivery locations. Not found: named leadership with profiles.", "ids": [ "ev-a1qa-003", "ev-a1qa-004", "ev-a1qa-002" ] } }, "grades": { "A": 0, "B": 14, "C": 47 }, "frameworksAB": [ "postman", "selenium", "cucumber-bdd", "jmeter", "rest-assured", "playwright", "detox", "gatling" ] }, { "slug": "thinksys", "name": "ThinkSys", "rank": 4, "ranked": true, "overall": 60, "dimensions": { "1": { "score": 7, "measured": "6.2 weighted cases", "rationale": "9 verified cases: A 2, B 7, C 0, weighted 1.0, 0.6 and 0.3.", "ids": [ "thinksys-1", "thinksys-2", "thinksys-3", "thinksys-4", "thinksys-5", "thinksys-6", "thinksys-7", "thinksys-8", "thinksys-9" ] }, "2": { "score": 6, "measured": "4 frameworks", "rationale": "Frameworks with at least one A or B case: Playwright, Selenium WebDriver, Cucumber, SpecFlow, Behave, Apache JMeter.", "ids": [ "thinksys-1", "thinksys-2", "thinksys-3", "thinksys-4", "thinksys-5", "thinksys-6", "thinksys-7", "thinksys-8", "thinksys-9" ] }, "3": { "score": 10, "measured": "7 cases on Playwright", "rationale": "Most cases on a single framework: Playwright.", "ids": [ "thinksys-1", "thinksys-2", "thinksys-3", "thinksys-4", "thinksys-5", "thinksys-6", "thinksys-7" ] }, "4": { "score": 7, "measured": "2 migration cases with before and after", "rationale": "2 cases tagged as migration; 2 state a before and an after value.", "ids": [ "thinksys-1", "thinksys-3" ] }, "5": { "score": 3, "measured": "6 of 27 metrics", "rationale": "6 of 27 published metrics state both a baseline and a method.", "ids": [ "thinksys-1", "thinksys-2" ] }, "6": { "score": 0, "measured": "0 reviews naming a framework", "rationale": "No verified review within 60 months names a listed framework.", "ids": [] }, "7": { "score": 6, "measured": "8 items", "rationale": "Dated posts, talks or repositories by staff about a listed framework within 36 months.", "ids": [ "ev-thinksys-005", "ev-thinksys-006", "ev-thinksys-007", "ev-thinksys-008", "ev-thinksys-009", "ev-thinksys-010", "ev-thinksys-011", "ev-thinksys-012" ] }, "8": { "score": 7.5, "measured": "3 of 4 items", "rationale": "Disclosed: public rate, QA process page, delivery locations. Not found: named leadership with profiles.", "ids": [ "ev-thinksys-001", "ev-thinksys-003" ] } }, "grades": { "A": 2, "B": 7, "C": 0 }, "frameworksAB": [ "playwright", "cucumber-bdd", "jmeter", "selenium" ] }, { "slug": "testfort", "name": "TestFort", "rank": 5, "ranked": true, "overall": 53, "dimensions": { "1": { "score": 9, "measured": "10.8 weighted cases", "rationale": "30 verified cases: A 0, B 6, C 24, weighted 1.0, 0.6 and 0.3.", "ids": [ "testfort-1", "testfort-2", "testfort-3", "testfort-4", "testfort-5", "testfort-6", "testfort-7", "testfort-8", "testfort-9", "testfort-10", "testfort-11", "testfort-12", "testfort-13", "testfort-14", "testfort-15", "testfort-16", "testfort-17", "testfort-18", "testfort-19", "testfort-20", "testfort-21", "testfort-22", "testfort-23", "testfort-24", "testfort-25", "testfort-26", "testfort-27", "testfort-28", "testfort-29", "testfort-30" ] }, "2": { "score": 8, "measured": "5 frameworks", "rationale": "Frameworks with at least one A or B case: Apache JMeter, Grafana k6, Selenium WebDriver, REST Assured, Postman and Newman.", "ids": [ "testfort-2", "testfort-3", "testfort-15", "testfort-4", "testfort-5", "testfort-8" ] }, "3": { "score": 10, "measured": "17 cases on Postman and Newman", "rationale": "Most cases on a single framework: Postman and Newman.", "ids": [ "testfort-6", "testfort-9", "testfort-10", "testfort-13", "testfort-14", "testfort-15", "testfort-16", "testfort-17", "testfort-18", "testfort-19", "testfort-20", "testfort-22", "testfort-23", "testfort-25", "testfort-27", "testfort-28", "testfort-29" ] }, "4": { "score": 0, "measured": "0 migration cases with before and after", "rationale": "0 cases tagged as migration; 0 state a before and an after value.", "ids": [] }, "5": { "score": 1, "measured": "1 of 108 metrics", "rationale": "1 of 108 published metrics state both a baseline and a method.", "ids": [ "testfort-8" ] }, "6": { "score": 0, "measured": "0 reviews naming a framework", "rationale": "No verified review within 60 months names a listed framework.", "ids": [] }, "7": { "score": 3, "measured": "2 items", "rationale": "Dated posts, talks or repositories by staff about a listed framework within 36 months.", "ids": [ "ev-testfort-004", "ev-testfort-005" ] }, "8": { "score": 7.5, "measured": "3 of 4 items", "rationale": "Disclosed: public rate, QA process page, delivery locations. Not found: named leadership with profiles.", "ids": [ "ev-testfort-001", "ev-testfort-003" ] } }, "grades": { "A": 0, "B": 6, "C": 24 }, "frameworksAB": [ "selenium", "jmeter", "k6", "postman", "rest-assured" ] }, { "slug": "abstracta", "name": "Abstracta", "rank": 6, "ranked": true, "overall": 50.5, "dimensions": { "1": { "score": 7, "measured": "4.2 weighted cases", "rationale": "7 verified cases: A 0, B 7, C 0, weighted 1.0, 0.6 and 0.3.", "ids": [ "abstracta-1", "abstracta-2", "abstracta-3", "abstracta-4", "abstracta-5", "abstracta-6", "abstracta-7" ] }, "2": { "score": 8, "measured": "6 frameworks", "rationale": "Frameworks with at least one A or B case: Appium, Playwright, WebdriverIO, Gatling, Apache JMeter, Selenium WebDriver.", "ids": [ "abstracta-1", "abstracta-2", "abstracta-3", "abstracta-4", "abstracta-5", "abstracta-6", "abstracta-7" ] }, "3": { "score": 4, "measured": "2 cases on Appium", "rationale": "Most cases on a single framework: Appium.", "ids": [ "abstracta-1", "abstracta-2" ] }, "4": { "score": 0, "measured": "0 migration cases with before and after", "rationale": "0 cases tagged as migration; 0 state a before and an after value.", "ids": [] }, "5": { "score": 1, "measured": "0 of 7 metrics", "rationale": "0 of 7 published metrics state both a baseline and a method.", "ids": [] }, "6": { "score": 3, "measured": "2 reviews naming a framework", "rationale": "Verified reviews within 60 months whose text names a listed framework.", "ids": [ "ev-abstracta-007", "ev-abstracta-008" ] }, "7": { "score": 8, "measured": "10 items", "rationale": "Dated posts, talks or repositories by staff about a listed framework within 36 months.", "ids": [ "ev-abstracta-013", "ev-abstracta-014", "ev-abstracta-015", "ev-abstracta-016", "ev-abstracta-017", "ev-abstracta-018", "ev-abstracta-019", "ev-abstracta-020", "ev-abstracta-021", "ev-abstracta-022" ] }, "8": { "score": 10, "measured": "4 of 4 items", "rationale": "Disclosed: public rate, named leadership with profiles, QA process page, delivery locations.", "ids": [ "ev-abstracta-001", "ev-abstracta-003", "ev-abstracta-006", "ev-abstracta-002" ] } }, "grades": { "A": 0, "B": 7, "C": 0 }, "frameworksAB": [ "appium", "jmeter", "selenium", "gatling", "playwright", "webdriverio" ] }, { "slug": "qawerk", "name": "QAwerk", "rank": 7, "ranked": true, "overall": 50.5, "dimensions": { "1": { "score": 7, "measured": "5.2 weighted cases", "rationale": "8 verified cases: A 1, B 7, C 0, weighted 1.0, 0.6 and 0.3.", "ids": [ "qawerk-1", "qawerk-2", "qawerk-3", "qawerk-4", "qawerk-5", "qawerk-6", "qawerk-7", "qawerk-8" ] }, "2": { "score": 10, "measured": "8 frameworks", "rationale": "Frameworks with at least one A or B case: Playwright, Appium, WebdriverIO, Postman and Newman, Apache JMeter, Puppeteer, Cucumber, SpecFlow, Behave, Cypress.", "ids": [ "qawerk-1", "qawerk-4", "qawerk-6", "qawerk-7", "qawerk-2", "qawerk-3", "qawerk-8", "qawerk-5" ] }, "3": { "score": 8, "measured": "4 cases on Playwright", "rationale": "Most cases on a single framework: Playwright.", "ids": [ "qawerk-1", "qawerk-4", "qawerk-6", "qawerk-7" ] }, "4": { "score": 0, "measured": "0 migration cases with before and after", "rationale": "1 case tagged as migration; 0 state a before and an after value.", "ids": [] }, "5": { "score": 1, "measured": "1 of 22 metrics", "rationale": "1 of 22 published metrics state both a baseline and a method.", "ids": [ "qawerk-5" ] }, "6": { "score": 0, "measured": "0 reviews naming a framework", "rationale": "No verified review within 60 months names a listed framework.", "ids": [] }, "7": { "score": 3, "measured": "2 items", "rationale": "Dated posts, talks or repositories by staff about a listed framework within 36 months.", "ids": [ "ev-qawerk-005", "ev-qawerk-006" ] }, "8": { "score": 10, "measured": "4 of 4 items", "rationale": "Disclosed: public rate, named leadership with profiles, QA process page, delivery locations.", "ids": [ "ev-qawerk-002", "ev-qawerk-007", "ev-qawerk-004", "ev-qawerk-003" ] } }, "grades": { "A": 1, "B": 7, "C": 0 }, "frameworksAB": [ "playwright", "postman", "appium", "cucumber-bdd", "cypress", "jmeter", "puppeteer", "webdriverio" ] }, { "slug": "qualitest", "name": "Qualitest", "rank": 8, "ranked": true, "overall": 44, "dimensions": { "1": { "score": 7, "measured": "4.5 weighted cases", "rationale": "12 verified cases: A 0, B 3, C 9, weighted 1.0, 0.6 and 0.3.", "ids": [ "qualitest-1", "qualitest-2", "qualitest-3", "qualitest-4", "qualitest-5", "qualitest-6", "qualitest-7", "qualitest-8", "qualitest-9", "qualitest-10", "qualitest-11", "qualitest-12" ] }, "2": { "score": 6, "measured": "3 frameworks", "rationale": "Frameworks with at least one A or B case: Selenium WebDriver, Robot Framework, Cucumber, SpecFlow, Behave.", "ids": [ "qualitest-4", "qualitest-5", "qualitest-11" ] }, "3": { "score": 10, "measured": "7 cases on Selenium WebDriver", "rationale": "Most cases on a single framework: Selenium WebDriver.", "ids": [ "qualitest-2", "qualitest-3", "qualitest-4", "qualitest-5", "qualitest-7", "qualitest-8", "qualitest-10" ] }, "4": { "score": 0, "measured": "0 migration cases with before and after", "rationale": "0 cases tagged as migration; 0 state a before and an after value.", "ids": [] }, "5": { "score": 1, "measured": "0 of 21 metrics", "rationale": "0 of 21 published metrics state both a baseline and a method.", "ids": [] }, "6": { "score": 0, "measured": "0 reviews naming a framework", "rationale": "No verified review within 60 months names a listed framework.", "ids": [] }, "7": { "score": 0, "measured": "0 items", "rationale": "No dated staff post, talk or repository about a listed framework within 36 months was verified.", "ids": [] }, "8": { "score": 7.5, "measured": "3 of 4 items", "rationale": "Disclosed: named leadership with profiles, QA process page, delivery locations. Not found: public rate.", "ids": [ "ev-qualitest-005", "ev-qualitest-006", "ev-qualitest-004" ] } }, "grades": { "A": 0, "B": 3, "C": 9 }, "frameworksAB": [ "selenium", "cucumber-bdd", "robot-framework" ] }, { "slug": "testrig", "name": "Testrig Technologies", "rank": 9, "ranked": true, "overall": 42, "dimensions": { "1": { "score": 5, "measured": "2.7 weighted cases", "rationale": "8 verified cases: A 0, B 1, C 7, weighted 1.0, 0.6 and 0.3.", "ids": [ "testrig-1", "testrig-3", "testrig-6", "testrig-7", "testrig-8", "testrig-9", "testrig-10", "testrig-11" ] }, "2": { "score": 2, "measured": "1 framework", "rationale": "Frameworks with at least one A or B case: Selenium WebDriver.", "ids": [ "testrig-7" ] }, "3": { "score": 6, "measured": "3 cases on Appium", "rationale": "Most cases on a single framework: Appium.", "ids": [ "testrig-6", "testrig-8", "testrig-10" ] }, "4": { "score": 4, "measured": "1 migration case with before and after", "rationale": "3 cases tagged as migration; 1 state a before and an after value.", "ids": [ "testrig-1" ] }, "5": { "score": 1, "measured": "0 of 6 metrics", "rationale": "0 of 6 published metrics state both a baseline and a method.", "ids": [] }, "6": { "score": 0, "measured": "0 reviews naming a framework", "rationale": "No verified review within 60 months names a listed framework.", "ids": [] }, "7": { "score": 8, "measured": "12 items", "rationale": "Dated posts, talks or repositories by staff about a listed framework within 36 months.", "ids": [ "ev-testrig-004", "ev-testrig-005", "ev-testrig-006", "ev-testrig-007", "ev-testrig-008", "ev-testrig-009", "ev-testrig-010", "ev-testrig-011", "ev-testrig-012", "ev-testrig-013", "ev-testrig-014", "ev-testrig-015" ] }, "8": { "score": 7.5, "measured": "3 of 4 items", "rationale": "Disclosed: public rate, QA process page, delivery locations. Not found: named leadership with profiles.", "ids": [ "ev-testrig-001", "ev-testrig-003", "ev-testrig-002" ] } }, "grades": { "A": 0, "B": 1, "C": 7 }, "frameworksAB": [ "selenium" ] }, { "slug": "zuci-systems", "name": "Zuci Systems", "rank": 10, "ranked": true, "overall": 38, "dimensions": { "1": { "score": 5, "measured": "3 weighted cases", "rationale": "8 verified cases: A 0, B 2, C 6, weighted 1.0, 0.6 and 0.3.", "ids": [ "zuci-systems-1", "zuci-systems-2", "zuci-systems-3", "zuci-systems-4", "zuci-systems-5", "zuci-systems-6", "zuci-systems-7", "zuci-systems-8" ] }, "2": { "score": 4, "measured": "2 frameworks", "rationale": "Frameworks with at least one A or B case: Selenium WebDriver, Cucumber, SpecFlow, Behave.", "ids": [ "zuci-systems-1", "zuci-systems-2" ] }, "3": { "score": 10, "measured": "7 cases on Selenium WebDriver", "rationale": "Most cases on a single framework: Selenium WebDriver.", "ids": [ "zuci-systems-1", "zuci-systems-2", "zuci-systems-4", "zuci-systems-5", "zuci-systems-6", "zuci-systems-7", "zuci-systems-8" ] }, "4": { "score": 0, "measured": "0 migration cases with before and after", "rationale": "0 cases tagged as migration; 0 state a before and an after value.", "ids": [] }, "5": { "score": 1, "measured": "0 of 27 metrics", "rationale": "0 of 27 published metrics state both a baseline and a method.", "ids": [] }, "6": { "score": 0, "measured": "0 reviews naming a framework", "rationale": "No verified review within 60 months names a listed framework.", "ids": [] }, "7": { "score": 0, "measured": "0 items", "rationale": "No dated staff post, talk or repository about a listed framework within 36 months was verified.", "ids": [] }, "8": { "score": 7.5, "measured": "3 of 4 items", "rationale": "Disclosed: public rate, named leadership with profiles, delivery locations. Not found: QA process page.", "ids": [ "ev-zuci-systems-004", "ev-zuci-systems-002", "ev-zuci-systems-003" ] } }, "grades": { "A": 0, "B": 2, "C": 6 }, "frameworksAB": [ "selenium", "cucumber-bdd" ] }, { "slug": "qa-wolf", "name": "QA Wolf", "rank": 11, "ranked": true, "overall": 36, "dimensions": { "1": { "score": 3, "measured": "1.2 weighted cases", "rationale": "2 verified cases: A 0, B 2, C 0, weighted 1.0, 0.6 and 0.3.", "ids": [ "qa-wolf-1", "qa-wolf-2" ] }, "2": { "score": 4, "measured": "2 frameworks", "rationale": "Frameworks with at least one A or B case: Playwright, Cypress.", "ids": [ "qa-wolf-1", "qa-wolf-2" ] }, "3": { "score": 4, "measured": "2 cases on Playwright", "rationale": "Most cases on a single framework: Playwright.", "ids": [ "qa-wolf-1", "qa-wolf-2" ] }, "4": { "score": 0, "measured": "0 migration cases with before and after", "rationale": "1 case tagged as migration; 0 state a before and an after value.", "ids": [] }, "5": { "score": 1, "measured": "0 of 7 metrics", "rationale": "0 of 7 published metrics state both a baseline and a method.", "ids": [] }, "6": { "score": 3, "measured": "1 review naming a framework", "rationale": "Verified reviews within 60 months whose text names a listed framework.", "ids": [ "ev-qa-wolf-002" ] }, "7": { "score": 8, "measured": "11 items", "rationale": "Dated posts, talks or repositories by staff about a listed framework within 36 months.", "ids": [ "ev-qa-wolf-008", "ev-qa-wolf-009", "ev-qa-wolf-010", "ev-qa-wolf-011", "ev-qa-wolf-012", "ev-qa-wolf-013", "ev-qa-wolf-014", "ev-qa-wolf-015", "ev-qa-wolf-016", "ev-qa-wolf-017", "ev-qa-wolf-018" ] }, "8": { "score": 7.5, "measured": "3 of 4 items", "rationale": "Disclosed: public rate, QA process page, delivery locations. Not found: named leadership with profiles.", "ids": [ "ev-qa-wolf-004", "ev-qa-wolf-005", "ev-qa-wolf-007" ] } }, "grades": { "A": 0, "B": 2, "C": 0 }, "frameworksAB": [ "playwright", "cypress" ] }, { "slug": "testdevlab", "name": "TestDevLab", "rank": 12, "ranked": true, "overall": 35, "dimensions": { "1": { "score": 5, "measured": "2.4 weighted cases", "rationale": "4 verified cases: A 0, B 4, C 0, weighted 1.0, 0.6 and 0.3.", "ids": [ "testdevlab-1", "testdevlab-2", "testdevlab-3", "testdevlab-4" ] }, "2": { "score": 8, "measured": "6 frameworks", "rationale": "Frameworks with at least one A or B case: Selenium WebDriver, Appium, Playwright, Cucumber, SpecFlow, Behave, Cypress, REST Assured.", "ids": [ "testdevlab-1", "testdevlab-4", "testdevlab-3", "testdevlab-2" ] }, "3": { "score": 6, "measured": "3 cases on Cucumber, SpecFlow, Behave", "rationale": "Most cases on a single framework: Cucumber, SpecFlow, Behave.", "ids": [ "testdevlab-2", "testdevlab-3", "testdevlab-4" ] }, "4": { "score": 0, "measured": "0 migration cases with before and after", "rationale": "0 cases tagged as migration; 0 state a before and an after value.", "ids": [] }, "5": { "score": 0, "measured": "0 metrics", "rationale": "The verified cases publish no metric.", "ids": [] }, "6": { "score": 0, "measured": "0 reviews naming a framework", "rationale": "No verified review within 60 months names a listed framework.", "ids": [] }, "7": { "score": 3, "measured": "3 items", "rationale": "Dated posts, talks or repositories by staff about a listed framework within 36 months.", "ids": [ "ev-testdevlab-004", "ev-testdevlab-005", "ev-testdevlab-006" ] }, "8": { "score": 5, "measured": "2 of 4 items", "rationale": "Disclosed: QA process page, delivery locations. Not found: public rate, named leadership with profiles.", "ids": [ "ev-testdevlab-003", "ev-testdevlab-001" ] } }, "grades": { "A": 0, "B": 4, "C": 0 }, "frameworksAB": [ "cucumber-bdd", "appium", "selenium", "cypress", "playwright", "rest-assured" ] }, { "slug": "testingxperts", "name": "TestingXperts", "rank": 13, "ranked": false, "overall": 34, "dimensions": { "1": { "score": 5, "measured": "3.6 weighted cases", "rationale": "10 verified cases: A 0, B 2, C 8, weighted 1.0, 0.6 and 0.3.", "ids": [ "testingxperts-1", "testingxperts-2", "testingxperts-3", "testingxperts-4", "testingxperts-5", "testingxperts-6", "testingxperts-7", "testingxperts-8", "testingxperts-9", "testingxperts-10" ] }, "2": { "score": 6, "measured": "3 frameworks", "rationale": "Frameworks with at least one A or B case: Playwright, Postman and Newman, Selenium WebDriver.", "ids": [ "testingxperts-3", "testingxperts-8" ] }, "3": { "score": 6, "measured": "3 cases on Playwright", "rationale": "Most cases on a single framework: Playwright.", "ids": [ "testingxperts-3", "testingxperts-9", "testingxperts-10" ] }, "4": { "score": 0, "measured": "0 migration cases with before and after", "rationale": "0 cases tagged as migration; 0 state a before and an after value.", "ids": [] }, "5": { "score": 1, "measured": "1 of 17 metrics", "rationale": "1 of 17 published metrics state both a baseline and a method.", "ids": [ "testingxperts-3" ] }, "6": { "score": 0, "measured": "0 reviews naming a framework", "rationale": "No verified review within 60 months names a listed framework.", "ids": [] }, "7": { "score": 0, "measured": "0 items", "rationale": "No dated staff post, talk or repository about a listed framework within 36 months was verified.", "ids": [] }, "8": { "score": 7.5, "measured": "3 of 4 items", "rationale": "Disclosed: public rate, QA process page, delivery locations. Not found: named leadership with profiles.", "ids": [ "ev-testingxperts-004", "ev-testingxperts-005", "ev-testingxperts-003" ] } }, "grades": { "A": 0, "B": 2, "C": 8 }, "frameworksAB": [ "playwright", "postman", "selenium" ] }, { "slug": "bugraptors", "name": "BugRaptors", "rank": 14, "ranked": false, "overall": 32, "dimensions": { "1": { "score": 3, "measured": "1.2 weighted cases", "rationale": "2 verified cases: A 0, B 2, C 0, weighted 1.0, 0.6 and 0.3.", "ids": [ "bugraptors-4", "bugraptors-6" ] }, "2": { "score": 2, "measured": "1 framework", "rationale": "Frameworks with at least one A or B case: Apache JMeter.", "ids": [ "bugraptors-4", "bugraptors-6" ] }, "3": { "score": 4, "measured": "2 cases on Apache JMeter", "rationale": "Most cases on a single framework: Apache JMeter.", "ids": [ "bugraptors-4", "bugraptors-6" ] }, "4": { "score": 0, "measured": "0 migration cases with before and after", "rationale": "0 cases tagged as migration; 0 state a before and an after value.", "ids": [] }, "5": { "score": 3, "measured": "1 of 9 metrics", "rationale": "1 of 9 published metrics state both a baseline and a method.", "ids": [ "bugraptors-6" ] }, "6": { "score": 0, "measured": "0 reviews naming a framework", "rationale": "No verified review within 60 months names a listed framework.", "ids": [] }, "7": { "score": 6, "measured": "6 items", "rationale": "Dated posts, talks or repositories by staff about a listed framework within 36 months.", "ids": [ "ev-bugraptors-005", "ev-bugraptors-006", "ev-bugraptors-007", "ev-bugraptors-008", "ev-bugraptors-009", "ev-bugraptors-010" ] }, "8": { "score": 7.5, "measured": "3 of 4 items", "rationale": "Disclosed: public rate, QA process page, delivery locations. Not found: named leadership with profiles.", "ids": [ "ev-bugraptors-002", "ev-bugraptors-004", "ev-bugraptors-001" ] } }, "grades": { "A": 0, "B": 2, "C": 0 }, "frameworksAB": [ "jmeter" ] }, { "slug": "qualitylogic", "name": "QualityLogic", "rank": 15, "ranked": false, "overall": 30, "dimensions": { "1": { "score": 3, "measured": "1.2 weighted cases", "rationale": "3 verified cases: A 0, B 1, C 2, weighted 1.0, 0.6 and 0.3.", "ids": [ "qualitylogic-1", "qualitylogic-2", "qualitylogic-3" ] }, "2": { "score": 2, "measured": "1 framework", "rationale": "Frameworks with at least one A or B case: Selenium WebDriver.", "ids": [ "qualitylogic-3" ] }, "3": { "score": 2, "measured": "1 cases on Grafana k6", "rationale": "Most cases on a single framework: Grafana k6.", "ids": [ "qualitylogic-2" ] }, "4": { "score": 4, "measured": "1 migration case with before and after", "rationale": "1 case tagged as migration; 1 state a before and an after value.", "ids": [ "qualitylogic-1" ] }, "5": { "score": 3, "measured": "1 of 8 metrics", "rationale": "1 of 8 published metrics state both a baseline and a method.", "ids": [ "qualitylogic-3" ] }, "6": { "score": 0, "measured": "0 reviews naming a framework", "rationale": "No verified review within 60 months names a listed framework.", "ids": [] }, "7": { "score": 3, "measured": "1 item", "rationale": "Dated posts, talks or repositories by staff about a listed framework within 36 months.", "ids": [ "ev-qualitylogic-007" ] }, "8": { "score": 7.5, "measured": "3 of 4 items", "rationale": "Disclosed: public rate, QA process page, delivery locations. Not found: named leadership with profiles.", "ids": [ "ev-qualitylogic-001", "ev-qualitylogic-005", "ev-qualitylogic-002" ] } }, "grades": { "A": 0, "B": 1, "C": 2 }, "frameworksAB": [ "selenium" ] }, { "slug": "impactqa", "name": "ImpactQA", "rank": 16, "ranked": false, "overall": 29, "dimensions": { "1": { "score": 5, "measured": "2.4 weighted cases", "rationale": "4 verified cases: A 0, B 4, C 0, weighted 1.0, 0.6 and 0.3.", "ids": [ "impactqa-1", "impactqa-2", "impactqa-3", "impactqa-4" ] }, "2": { "score": 4, "measured": "2 frameworks", "rationale": "Frameworks with at least one A or B case: Selenium WebDriver, Apache JMeter.", "ids": [ "impactqa-1", "impactqa-4", "impactqa-2", "impactqa-3" ] }, "3": { "score": 4, "measured": "2 cases on Apache JMeter", "rationale": "Most cases on a single framework: Apache JMeter.", "ids": [ "impactqa-2", "impactqa-3" ] }, "4": { "score": 0, "measured": "0 migration cases with before and after", "rationale": "0 cases tagged as migration; 0 state a before and an after value.", "ids": [] }, "5": { "score": 1, "measured": "0 of 6 metrics", "rationale": "0 of 6 published metrics state both a baseline and a method.", "ids": [] }, "6": { "score": 0, "measured": "0 reviews naming a framework", "rationale": "No verified review within 60 months names a listed framework.", "ids": [] }, "7": { "score": 0, "measured": "0 items", "rationale": "No dated staff post, talk or repository about a listed framework within 36 months was verified.", "ids": [] }, "8": { "score": 7.5, "measured": "3 of 4 items", "rationale": "Disclosed: public rate, QA process page, delivery locations. Not found: named leadership with profiles.", "ids": [ "ev-impactqa-001", "ev-impactqa-004", "ev-impactqa-002" ] } }, "grades": { "A": 0, "B": 4, "C": 0 }, "frameworksAB": [ "jmeter", "selenium" ] }, { "slug": "qa-madness", "name": "QA Madness", "rank": 17, "ranked": false, "overall": 14.5, "dimensions": { "1": { "score": 1, "measured": "0.6 weighted cases", "rationale": "2 verified cases: A 0, B 0, C 2, weighted 1.0, 0.6 and 0.3.", "ids": [ "qa-madness-1", "qa-madness-2" ] }, "2": { "score": 0, "measured": "0 frameworks", "rationale": "No framework has an A or B case.", "ids": [] }, "3": { "score": 2, "measured": "1 cases on Cypress", "rationale": "Most cases on a single framework: Cypress.", "ids": [ "qa-madness-1" ] }, "4": { "score": 0, "measured": "0 migration cases with before and after", "rationale": "0 cases tagged as migration; 0 state a before and an after value.", "ids": [] }, "5": { "score": 1, "measured": "0 of 2 metrics", "rationale": "0 of 2 published metrics state both a baseline and a method.", "ids": [] }, "6": { "score": 0, "measured": "0 reviews naming a framework", "rationale": "No verified review within 60 months names a listed framework.", "ids": [] }, "7": { "score": 3, "measured": "2 items", "rationale": "Dated posts, talks or repositories by staff about a listed framework within 36 months.", "ids": [ "ev-qa-madness-005", "ev-qa-madness-006" ] }, "8": { "score": 5, "measured": "2 of 4 items", "rationale": "Disclosed: public rate, delivery locations. Not found: named leadership with profiles, QA process page.", "ids": [ "ev-qa-madness-001" ] } }, "grades": { "A": 0, "B": 0, "C": 2 }, "frameworksAB": [] } ] }