diff options
| -rw-r--r-- | .github/workflows/ci_ssl.yml | 93 | ||||
| -rw-r--r-- | .gitlab-ci.yml | 2 | ||||
| -rw-r--r-- | koch.nim | 2 | ||||
| -rw-r--r-- | testament/categories.nim | 7 | ||||
| -rw-r--r-- | testament/lib/stdtest/testutils.nim | 8 | ||||
| -rw-r--r-- | testament/testament.nim | 5 | ||||
| -rw-r--r-- | tests/config.nims | 4 | ||||
| -rw-r--r-- | tests/misc/trunner.nim | 7 | ||||
| -rw-r--r-- | tests/misc/trunner_special.nim | 31 | ||||
| -rw-r--r-- | tests/untestable/thttpclient_ssl.nim | 207 | ||||
| -rw-r--r-- | tests/untestable/thttpclient_ssl_disabled.nim | 46 | ||||
| -rw-r--r-- | tests/untestable/thttpclient_ssl_remotenetwork.nim | 209 |
12 files changed, 285 insertions, 336 deletions
diff --git a/.github/workflows/ci_ssl.yml b/.github/workflows/ci_ssl.yml deleted file mode 100644 index 0ec7aeb3b..000000000 --- a/.github/workflows/ci_ssl.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: Nim SSL CI -on: - pull_request: - # Run only on changes on related files - paths: - - 'lib/pure/httpclient.nim' - - 'lib/pure/net.nim' - - 'lib/pure/ssl_certs.nim' - - 'lib/wrappers/openssl.nim' - - 'tests/stdlib/thttpclient_ssl*' - - 'tests/untestable/thttpclient_ssl*' - -jobs: - build: - if: | - !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]') - strategy: - fail-fast: false - matrix: - os: [ubuntu-18.04, macos-10.15, windows-2019] - cpu: [amd64] - name: '${{ matrix.os }} (${{ matrix.cpu }})' - runs-on: ${{ matrix.os }} - steps: - - name: 'Checkout' - uses: actions/checkout@v2 - - - name: 'Checkout csources' - uses: actions/checkout@v2 - with: - repository: nim-lang/csources - path: csources - - - name: 'Install dependencies (Linux amd64)' - if: runner.os == 'Linux' && matrix.cpu == 'amd64' - run: | - sudo apt-fast update -qq - DEBIAN_FRONTEND='noninteractive' \ - sudo apt-fast install --no-install-recommends -y libssl1.1 - - name: 'Install dependencies (macOS)' - if: runner.os == 'macOS' - run: brew install make - - name: 'Install dependencies (Windows)' - if: runner.os == 'Windows' - shell: bash - run: | - mkdir dist - curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z - curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip - 7z x dist/mingw64.7z -odist - 7z x dist/dlls.zip -obin - echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}" - - - name: 'Add build binaries to PATH' - shell: bash - run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}" - - - name: 'Build 1-stage compiler from csources' - shell: bash - run: | - ncpu= - case '${{ runner.os }}' in - 'Linux') - ncpu=$(nproc) - ;; - 'macOS') - ncpu=$(sysctl -n hw.ncpu) - ;; - 'Windows') - ncpu=$NUMBER_OF_PROCESSORS - ;; - esac - [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1 - - make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}' - - - name: 'Build koch' - shell: bash - run: nim c koch - - - name: 'Build the real compiler' - shell: bash - run: ./koch boot - - - name: 'Run SSL nimDisableCertificateValidation integration tests' - shell: bash - run: nim c -d:nimDisableCertificateValidation -d:ssl -r -p:. tests/untestable/thttpclient_ssl_disabled.nim - - - name: 'Run SSL certificate check integration tests' - # Not supported on Windows due to old openssl version - if: runner.os != 'Windows' - shell: bash - run: nim c -d:ssl -p:. --threads:on -r tests/untestable/thttpclient_ssl.nim diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9f536e6c9..ad9e565a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -54,7 +54,7 @@ test-windows: script: - call ci\deps.bat - nim c testament\tester - - testament\tester.exe --pedantic all + - testament\tester.exe all tags: - windows - fast diff --git a/koch.nim b/koch.nim index 95c9c6161..6057f8cff 100644 --- a/koch.nim +++ b/koch.nim @@ -548,7 +548,7 @@ proc runCI(cmd: string) = #[ BUG: with initOptParser, `--batch:'' all` interprets `all` as the argument of --batch ]# - execFold("Run tester", "nim c -r -d:nimCoroutines testament/testament --pedantic --batch:$1 all -d:nimCoroutines" % ["NIM_TESTAMENT_BATCH".getEnv("_")]) + execFold("Run tester", "nim c -r -d:nimCoroutines --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 testament/testament --batch:$1 all -d:nimCoroutines" % ["NIM_TESTAMENT_BATCH".getEnv("_")]) block CT_FFI: when defined(posix): # windows can be handled in future PR's diff --git a/testament/categories.nim b/testament/categories.nim index 9b0de13ac..e1db9aa10 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -34,7 +34,7 @@ const "nimble-packages-2", "niminaction", "threads", - "untestable", + "untestable", # see trunner_special "testdata", "nimcache", "coroutines", @@ -711,8 +711,9 @@ proc processCategory(r: var TResults, cat: Category, of "ic": icTests(r, testsDir, cat, options) of "untestable": - # We can't test it because it depends on a third party. - discard # TODO: Move untestable tests to someplace else, i.e. nimble repo. + # These require special treatment e.g. because they depend on a third party + # dependency; see `trunner_special` which runs some of those. + discard else: handled = false if not handled: diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim index 34aa3b751..50dda22e2 100644 --- a/testament/lib/stdtest/testutils.nim +++ b/testament/lib/stdtest/testutils.nim @@ -1,5 +1,6 @@ import std/private/miscdollars import std/strutils +from std/os import getEnv template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) = ## API to deal with flaky or failing tests. This avoids disabling entire tests @@ -34,3 +35,10 @@ proc greedyOrderedSubsetLines*(lhs, rhs: string): bool = if currentPos < 0: return false return true + +template enableRemoteNetworking*: bool = + ## Allows contolling whether to run some test at a statement-level granularity. + ## Using environment variables simplifies propagating this all the way across + ## process calls, e.g. `testament all` calls itself, which in turns invokes + ## a `nim` invocation (possibly via additional intermediate processes). + getEnv("NIM_TESTAMENT_REMOTE_NETWORKING") == "1" diff --git a/testament/testament.nim b/testament/testament.nim index c6ceed224..2ee1539ff 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -52,6 +52,9 @@ Options: On Azure Pipelines, testament will also publish test results via Azure Pipelines' Test Management API provided that System.AccessToken is made available via the environment variable SYSTEM_ACCESSTOKEN. + +Experimental: using environment variable `NIM_TESTAMENT_REMOTE_NETWORKING=1` enables +tests with remote networking (as in CI). """ % resultsFile proc isNimRepoTests(): bool = @@ -680,7 +683,7 @@ proc main() = case p.key.normalize of "print", "verbose": optPrintResults = true of "failing": optFailing = true - of "pedantic": discard "now always enabled" + of "pedantic": discard # deadcode of "targets": targetsStr = p.val gTargets = parseTargets(targetsStr) diff --git a/tests/config.nims b/tests/config.nims index e330e097c..b7d828183 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -17,3 +17,7 @@ switch("define", "nimLegacyTypeMismatch") hint("Processing", off) # dots can cause annoyances; instead, a single test can test `hintProcessing` + +# uncomment to enable all flaky tests disabled by this flag +# (works through process calls, e.g. tests that invoke nim). +# switch("define", "nimTestsEnableFlaky") diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 63aac0ede..a66177f28 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -7,7 +7,7 @@ discard """ ## A few others could be added to here to simplify code. ## Note: this test is a bit slow but tests a lot of things; please don't disable. -import std/[strformat,os,osproc,unittest] +import std/[strformat,os,osproc,unittest,compilesettings] from std/sequtils import toSeq,mapIt from std/algorithm import sorted import stdtest/[specialpaths, unittest_light] @@ -25,10 +25,7 @@ const # useful when you want to turn only some hints on, and some common ones off. # pending https://github.com/timotheecour/Nim/issues/453, simplify to: `--hints:off` nim = getCurrentCompilerExe() - mode = - when defined(c): "c" - elif defined(cpp): "cpp" - else: static: doAssert false + mode = querySetting(backend) nimcache = buildDir / "nimcacheTrunner" # instead of `querySetting(nimcacheDir)`, avoids stomping on other parallel tests diff --git a/tests/misc/trunner_special.nim b/tests/misc/trunner_special.nim new file mode 100644 index 000000000..47ba44a29 --- /dev/null +++ b/tests/misc/trunner_special.nim @@ -0,0 +1,31 @@ +discard """ + targets: "c cpp" + joinable: false +""" + +#[ +Runs tests that require special treatment, e.g. because they rely on 3rd party code +or require external networking. + +xxx test all tests/untestable/* here, possibly with adjustments to make running times reasonable +]# + +import std/[strformat,os,unittest,compilesettings] +import stdtest/specialpaths + +const + nim = getCurrentCompilerExe() + mode = querySetting(backend) + +proc runCmd(cmd: string) = + let ret = execShellCmd(cmd) + check ret == 0 # allows more than 1 failure + +proc main = + let options = fmt"-b:{mode} --hints:off" + block: # SSL nimDisableCertificateValidation integration tests + runCmd fmt"{nim} r {options} -d:nimDisableCertificateValidation -d:ssl {testsDir}/untestable/thttpclient_ssl_disabled.nim" + block: # SSL certificate check integration tests + runCmd fmt"{nim} r {options} -d:ssl --threads:on {testsDir}/untestable/thttpclient_ssl_remotenetwork.nim" + +main() diff --git a/tests/untestable/thttpclient_ssl.nim b/tests/untestable/thttpclient_ssl.nim deleted file mode 100644 index 038eba99d..000000000 --- a/tests/untestable/thttpclient_ssl.nim +++ /dev/null @@ -1,207 +0,0 @@ -# -# -# Nim - SSL integration tests -# (c) Copyright 2017 Nim contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# -## Warning: this test performs external networking. -## Test with: -## ./bin/nim c -d:ssl -p:. --threads:on -r tests/untestable/thttpclient_ssl.nim -## -## See https://github.com/FedericoCeratto/ssl-comparison/blob/master/README.md -## for a comparison with other clients. - -import - httpclient, - net, - strutils, - threadpool, - unittest - - -type - # bad and dubious tests should not pass SSL validation - # "_broken" mark the test as skipped. Some tests have different - # behavior depending on OS and SSL version! - # TODO: chase and fix the broken tests - Category = enum - good, bad, dubious, good_broken, bad_broken, dubious_broken - CertTest = tuple[url:string, category:Category, desc: string] - -const certificate_tests: array[0..55, CertTest] = [ - ("https://wrong.host.badssl.com/", bad, "wrong.host"), - ("https://captive-portal.badssl.com/", bad, "captive-portal"), - ("https://expired.badssl.com/", bad, "expired"), - ("https://google.com/", good, "good"), - ("https://self-signed.badssl.com/", bad, "self-signed"), - ("https://untrusted-root.badssl.com/", bad, "untrusted-root"), - ("https://revoked.badssl.com/", bad_broken, "revoked"), - ("https://pinning-test.badssl.com/", bad_broken, "pinning-test"), - ("https://no-common-name.badssl.com/", dubious_broken, "no-common-name"), - ("https://no-subject.badssl.com/", dubious_broken, "no-subject"), - ("https://incomplete-chain.badssl.com/", dubious_broken, "incomplete-chain"), - ("https://sha1-intermediate.badssl.com/", bad, "sha1-intermediate"), - ("https://sha256.badssl.com/", good, "sha256"), - ("https://sha384.badssl.com/", good, "sha384"), - ("https://sha512.badssl.com/", good, "sha512"), - ("https://1000-sans.badssl.com/", good, "1000-sans"), - ("https://10000-sans.badssl.com/", good_broken, "10000-sans"), - ("https://ecc256.badssl.com/", good, "ecc256"), - ("https://ecc384.badssl.com/", good, "ecc384"), - ("https://rsa2048.badssl.com/", good, "rsa2048"), - ("https://rsa8192.badssl.com/", dubious_broken, "rsa8192"), - ("http://http.badssl.com/", good, "regular http"), - ("https://http.badssl.com/", bad_broken, "http on https URL"), # FIXME - ("https://cbc.badssl.com/", dubious, "cbc"), - ("https://rc4-md5.badssl.com/", bad, "rc4-md5"), - ("https://rc4.badssl.com/", bad, "rc4"), - ("https://3des.badssl.com/", bad, "3des"), - ("https://null.badssl.com/", bad, "null"), - ("https://mozilla-old.badssl.com/", bad_broken, "mozilla-old"), - ("https://mozilla-intermediate.badssl.com/", dubious_broken, "mozilla-intermediate"), - ("https://mozilla-modern.badssl.com/", good, "mozilla-modern"), - ("https://dh480.badssl.com/", bad, "dh480"), - ("https://dh512.badssl.com/", bad, "dh512"), - ("https://dh1024.badssl.com/", dubious_broken, "dh1024"), - ("https://dh2048.badssl.com/", good, "dh2048"), - ("https://dh-small-subgroup.badssl.com/", bad_broken, "dh-small-subgroup"), - ("https://dh-composite.badssl.com/", bad_broken, "dh-composite"), - ("https://static-rsa.badssl.com/", dubious, "static-rsa"), - ("https://tls-v1-0.badssl.com:1010/", dubious, "tls-v1-0"), - ("https://tls-v1-1.badssl.com:1011/", dubious, "tls-v1-1"), - ("https://invalid-expected-sct.badssl.com/", bad, "invalid-expected-sct"), - ("https://hsts.badssl.com/", good, "hsts"), - ("https://upgrade.badssl.com/", good, "upgrade"), - ("https://preloaded-hsts.badssl.com/", good, "preloaded-hsts"), - ("https://subdomain.preloaded-hsts.badssl.com/", bad, "subdomain.preloaded-hsts"), - ("https://https-everywhere.badssl.com/", good, "https-everywhere"), - ("https://long-extended-subdomain-name-containing-many-letters-and-dashes.badssl.com/", good, - "long-extended-subdomain-name-containing-many-letters-and-dashes"), - ("https://longextendedsubdomainnamewithoutdashesinordertotestwordwrapping.badssl.com/", good, - "longextendedsubdomainnamewithoutdashesinordertotestwordwrapping"), - ("https://superfish.badssl.com/", bad, "(Lenovo) Superfish"), - ("https://edellroot.badssl.com/", bad, "(Dell) eDellRoot"), - ("https://dsdtestprovider.badssl.com/", bad, "(Dell) DSD Test Provider"), - ("https://preact-cli.badssl.com/", bad, "preact-cli"), - ("https://webpack-dev-server.badssl.com/", bad, "webpack-dev-server"), - ("https://mitm-software.badssl.com/", bad, "mitm-software"), - ("https://sha1-2016.badssl.com/", dubious, "sha1-2016"), - ("https://sha1-2017.badssl.com/", bad, "sha1-2017"), -] - - -template evaluate(exception_msg: string, category: Category, desc: string) = - # Evaluate test outcome. Testes flagged as _broken are evaluated and skipped - let raised = (exception_msg.len > 0) - let should_not_raise = category in {good, dubious_broken, bad_broken} - if should_not_raise xor raised: - # we are seeing a known behavior - if category in {good_broken, dubious_broken, bad_broken}: - skip() - if raised: - # check exception_msg == "No SSL certificate found." or - doAssert exception_msg == "No SSL certificate found." or - exception_msg == "SSL Certificate check failed." or - exception_msg.contains("certificate verify failed") or - exception_msg.contains("key too small") or - exception_msg.contains("alert handshake failure") or - exception_msg.contains("bad dh p length") or - # TODO: This one should only triggers for 10000-sans - exception_msg.contains("excessive message size"), exception_msg - - else: - # this is unexpected - if raised: - echo " $# ($#) raised: $#" % [desc, $category, exception_msg] - else: - echo " $# ($#) did not raise" % [desc, $category] - if category in {good, dubious, bad}: - fail() - - -suite "SSL certificate check - httpclient": - - for i, ct in certificate_tests: - - test ct.desc: - var ctx = newContext(verifyMode=CVerifyPeer) - var client = newHttpClient(sslContext=ctx) - let exception_msg = - try: - let a = $client.getContent(ct.url) - "" - except: - getCurrentExceptionMsg() - - evaluate(exception_msg, ct.category, ct.desc) - - - -# threaded tests - - -type - TTOutcome = ref object - desc, exception_msg: string - category: Category - -proc run_t_test(ct: CertTest): TTOutcome {.thread.} = - ## Run test in a {.thread.} - return by ref - result = TTOutcome(desc:ct.desc, exception_msg:"", category: ct.category) - try: - var ctx = newContext(verifyMode=CVerifyPeer) - var client = newHttpClient(sslContext=ctx) - let a = $client.getContent(ct.url) - except: - result.exception_msg = getCurrentExceptionMsg() - - -suite "SSL certificate check - httpclient - threaded": - - # Spawn threads before the "test" blocks - var outcomes = newSeq[FlowVar[TTOutcome]](certificate_tests.len) - for i, ct in certificate_tests: - let t = spawn run_t_test(ct) - outcomes[i] = t - - # create "test" blocks and handle thread outputs - for t in outcomes: - let outcome = ^t # wait for a thread to terminate - - test outcome.desc: - - evaluate(outcome.exception_msg, outcome.category, outcome.desc) - - -# net tests - - -type NetSocketTest = tuple[hostname: string, port: Port, category:Category, desc: string] -const net_tests:array[0..3, NetSocketTest] = [ - ("imap.gmail.com", 993.Port, good, "IMAP"), - ("wrong.host.badssl.com", 443.Port, bad, "wrong.host"), - ("captive-portal.badssl.com", 443.Port, bad, "captive-portal"), - ("expired.badssl.com", 443.Port, bad, "expired"), -] -# TODO: ("null.badssl.com", 443.Port, bad_broken, "null"), - - -suite "SSL certificate check - sockets": - - for ct in net_tests: - - test ct.desc: - - var sock = newSocket() - var ctx = newContext() - ctx.wrapSocket(sock) - let exception_msg = - try: - sock.connect(ct.hostname, ct.port) - "" - except: - getCurrentExceptionMsg() - - evaluate(exception_msg, ct.category, ct.desc) diff --git a/tests/untestable/thttpclient_ssl_disabled.nim b/tests/untestable/thttpclient_ssl_disabled.nim index 0f7e95831..b95dad2c6 100644 --- a/tests/untestable/thttpclient_ssl_disabled.nim +++ b/tests/untestable/thttpclient_ssl_disabled.nim @@ -5,36 +5,32 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. # -## Warning: this test performs external networking. ## Compile and run with: -## ./bin/nim c -d:nimDisableCertificateValidation -d:ssl -r -p:. tests/untestable/thttpclient_ssl_disabled.nim +## nim r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimDisableCertificateValidation -d:ssl -p:. tests/untestable/thttpclient_ssl_disabled.nim -import httpclient, - net, - unittest, - os +from stdtest/testutils import enableRemoteNetworking +when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(openbsd)): + import httpclient, net, unittest -from strutils import contains + const expired = "https://expired.badssl.com/" -const expired = "https://expired.badssl.com/" + doAssert defined(nimDisableCertificateValidation) -doAssert defined(nimDisableCertificateValidation) + suite "SSL certificate check - disabled": -suite "SSL certificate check - disabled": + test "httpclient in insecure mode": + var ctx = newContext(verifyMode = CVerifyPeer) + var client = newHttpClient(sslContext = ctx) + let a = $client.getContent(expired) - test "httpclient in insecure mode": - var ctx = newContext(verifyMode = CVerifyPeer) - var client = newHttpClient(sslContext = ctx) - let a = $client.getContent(expired) + test "httpclient in insecure mode": + var ctx = newContext(verifyMode = CVerifyPeerUseEnvVars) + var client = newHttpClient(sslContext = ctx) + let a = $client.getContent(expired) - test "httpclient in insecure mode": - var ctx = newContext(verifyMode = CVerifyPeerUseEnvVars) - var client = newHttpClient(sslContext = ctx) - let a = $client.getContent(expired) - - test "net socket in insecure mode": - var sock = newSocket() - var ctx = newContext(verifyMode = CVerifyPeerUseEnvVars) - ctx.wrapSocket(sock) - sock.connect("expired.badssl.com", 443.Port) - sock.close + test "net socket in insecure mode": + var sock = newSocket() + var ctx = newContext(verifyMode = CVerifyPeerUseEnvVars) + ctx.wrapSocket(sock) + sock.connect("expired.badssl.com", 443.Port) + sock.close diff --git a/tests/untestable/thttpclient_ssl_remotenetwork.nim b/tests/untestable/thttpclient_ssl_remotenetwork.nim new file mode 100644 index 000000000..ed6866d53 --- /dev/null +++ b/tests/untestable/thttpclient_ssl_remotenetwork.nim @@ -0,0 +1,209 @@ +# +# +# Nim - SSL integration tests +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# +## Test with: +## nim r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:ssl -p:. --threads:on tests/untestable/thttpclient_ssl_remotenetwork.nim +## +## See https://github.com/FedericoCeratto/ssl-comparison/blob/master/README.md +## for a comparison with other clients. + +from stdtest/testutils import enableRemoteNetworking +when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(windows) and not defined(openbsd) and not defined(i386)): + # Not supported on Windows due to old openssl version + import + httpclient, + net, + strutils, + threadpool, + unittest + + + type + # bad and dubious tests should not pass SSL validation + # "_broken" mark the test as skipped. Some tests have different + # behavior depending on OS and SSL version! + # TODO: chase and fix the broken tests + Category = enum + good, bad, dubious, good_broken, bad_broken, dubious_broken + CertTest = tuple[url:string, category:Category, desc: string] + + const certificate_tests: array[0..55, CertTest] = [ + ("https://wrong.host.badssl.com/", bad, "wrong.host"), + ("https://captive-portal.badssl.com/", bad, "captive-portal"), + ("https://expired.badssl.com/", bad, "expired"), + ("https://google.com/", good, "good"), + ("https://self-signed.badssl.com/", bad, "self-signed"), + ("https://untrusted-root.badssl.com/", bad, "untrusted-root"), + ("https://revoked.badssl.com/", bad_broken, "revoked"), + ("https://pinning-test.badssl.com/", bad_broken, "pinning-test"), + ("https://no-common-name.badssl.com/", dubious_broken, "no-common-name"), + ("https://no-subject.badssl.com/", dubious_broken, "no-subject"), + ("https://incomplete-chain.badssl.com/", dubious_broken, "incomplete-chain"), + ("https://sha1-intermediate.badssl.com/", bad, "sha1-intermediate"), + ("https://sha256.badssl.com/", good, "sha256"), + ("https://sha384.badssl.com/", good, "sha384"), + ("https://sha512.badssl.com/", good, "sha512"), + ("https://1000-sans.badssl.com/", good, "1000-sans"), + ("https://10000-sans.badssl.com/", good_broken, "10000-sans"), + ("https://ecc256.badssl.com/", good, "ecc256"), + ("https://ecc384.badssl.com/", good, "ecc384"), + ("https://rsa2048.badssl.com/", good, "rsa2048"), + ("https://rsa8192.badssl.com/", dubious_broken, "rsa8192"), + ("http://http.badssl.com/", good, "regular http"), + ("https://http.badssl.com/", bad_broken, "http on https URL"), # FIXME + ("https://cbc.badssl.com/", dubious, "cbc"), + ("https://rc4-md5.badssl.com/", bad, "rc4-md5"), + ("https://rc4.badssl.com/", bad, "rc4"), + ("https://3des.badssl.com/", bad, "3des"), + ("https://null.badssl.com/", bad, "null"), + ("https://mozilla-old.badssl.com/", bad_broken, "mozilla-old"), + ("https://mozilla-intermediate.badssl.com/", dubious_broken, "mozilla-intermediate"), + ("https://mozilla-modern.badssl.com/", good, "mozilla-modern"), + ("https://dh480.badssl.com/", bad, "dh480"), + ("https://dh512.badssl.com/", bad, "dh512"), + ("https://dh1024.badssl.com/", dubious_broken, "dh1024"), + ("https://dh2048.badssl.com/", good, "dh2048"), + ("https://dh-small-subgroup.badssl.com/", bad_broken, "dh-small-subgroup"), + ("https://dh-composite.badssl.com/", bad_broken, "dh-composite"), + ("https://static-rsa.badssl.com/", dubious, "static-rsa"), + ("https://tls-v1-0.badssl.com:1010/", dubious, "tls-v1-0"), + ("https://tls-v1-1.badssl.com:1011/", dubious, "tls-v1-1"), + ("https://invalid-expected-sct.badssl.com/", bad, "invalid-expected-sct"), + ("https://hsts.badssl.com/", good, "hsts"), + ("https://upgrade.badssl.com/", good, "upgrade"), + ("https://preloaded-hsts.badssl.com/", good, "preloaded-hsts"), + ("https://subdomain.preloaded-hsts.badssl.com/", bad, "subdomain.preloaded-hsts"), + ("https://https-everywhere.badssl.com/", good, "https-everywhere"), + ("https://long-extended-subdomain-name-containing-many-letters-and-dashes.badssl.com/", good, + "long-extended-subdomain-name-containing-many-letters-and-dashes"), + ("https://longextendedsubdomainnamewithoutdashesinordertotestwordwrapping.badssl.com/", good, + "longextendedsubdomainnamewithoutdashesinordertotestwordwrapping"), + ("https://superfish.badssl.com/", bad, "(Lenovo) Superfish"), + ("https://edellroot.badssl.com/", bad, "(Dell) eDellRoot"), + ("https://dsdtestprovider.badssl.com/", bad, "(Dell) DSD Test Provider"), + ("https://preact-cli.badssl.com/", bad, "preact-cli"), + ("https://webpack-dev-server.badssl.com/", bad, "webpack-dev-server"), + ("https://mitm-software.badssl.com/", bad, "mitm-software"), + ("https://sha1-2016.badssl.com/", dubious, "sha1-2016"), + ("https://sha1-2017.badssl.com/", bad, "sha1-2017"), + ] + + + template evaluate(exception_msg: string, category: Category, desc: string) = + # Evaluate test outcome. Testes flagged as _broken are evaluated and skipped + let raised = (exception_msg.len > 0) + let should_not_raise = category in {good, dubious_broken, bad_broken} + if should_not_raise xor raised: + # we are seeing a known behavior + if category in {good_broken, dubious_broken, bad_broken}: + skip() + if raised: + # check exception_msg == "No SSL certificate found." or + doAssert exception_msg == "No SSL certificate found." or + exception_msg == "SSL Certificate check failed." or + exception_msg.contains("certificate verify failed") or + exception_msg.contains("key too small") or + exception_msg.contains("alert handshake failure") or + exception_msg.contains("bad dh p length") or + # TODO: This one should only triggers for 10000-sans + exception_msg.contains("excessive message size"), exception_msg + + else: + # this is unexpected + if raised: + echo " $# ($#) raised: $#" % [desc, $category, exception_msg] + else: + echo " $# ($#) did not raise" % [desc, $category] + if category in {good, dubious, bad}: + fail() + + + suite "SSL certificate check - httpclient": + + for i, ct in certificate_tests: + + test ct.desc: + var ctx = newContext(verifyMode=CVerifyPeer) + var client = newHttpClient(sslContext=ctx) + let exception_msg = + try: + let a = $client.getContent(ct.url) + "" + except: + getCurrentExceptionMsg() + + evaluate(exception_msg, ct.category, ct.desc) + + + + # threaded tests + + + type + TTOutcome = ref object + desc, exception_msg: string + category: Category + + proc run_t_test(ct: CertTest): TTOutcome {.thread.} = + ## Run test in a {.thread.} - return by ref + result = TTOutcome(desc:ct.desc, exception_msg:"", category: ct.category) + try: + var ctx = newContext(verifyMode=CVerifyPeer) + var client = newHttpClient(sslContext=ctx) + let a = $client.getContent(ct.url) + except: + result.exception_msg = getCurrentExceptionMsg() + + + suite "SSL certificate check - httpclient - threaded": + + # Spawn threads before the "test" blocks + var outcomes = newSeq[FlowVar[TTOutcome]](certificate_tests.len) + for i, ct in certificate_tests: + let t = spawn run_t_test(ct) + outcomes[i] = t + + # create "test" blocks and handle thread outputs + for t in outcomes: + let outcome = ^t # wait for a thread to terminate + + test outcome.desc: + + evaluate(outcome.exception_msg, outcome.category, outcome.desc) + + + # net tests + + + type NetSocketTest = tuple[hostname: string, port: Port, category:Category, desc: string] + const net_tests:array[0..3, NetSocketTest] = [ + ("imap.gmail.com", 993.Port, good, "IMAP"), + ("wrong.host.badssl.com", 443.Port, bad, "wrong.host"), + ("captive-portal.badssl.com", 443.Port, bad, "captive-portal"), + ("expired.badssl.com", 443.Port, bad, "expired"), + ] + # TODO: ("null.badssl.com", 443.Port, bad_broken, "null"), + + + suite "SSL certificate check - sockets": + + for ct in net_tests: + + test ct.desc: + + var sock = newSocket() + var ctx = newContext() + ctx.wrapSocket(sock) + let exception_msg = + try: + sock.connect(ct.hostname, ct.port) + "" + except: + getCurrentExceptionMsg() + + evaluate(exception_msg, ct.category, ct.desc) |