diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/config.nims | 4 | ||||
| -rw-r--r-- | tests/misc/trunner.nim | 7 | ||||
| -rw-r--r-- | tests/misc/trunner_special.nim | 31 | ||||
| -rw-r--r-- | tests/untestable/thttpclient_ssl.nim | 207 | ||||
| -rw-r--r-- | tests/untestable/thttpclient_ssl_disabled.nim | 46 | ||||
| -rw-r--r-- | tests/untestable/thttpclient_ssl_remotenetwork.nim | 209 |
6 files changed, 267 insertions, 237 deletions
diff --git a/tests/config.nims b/tests/config.nims index e330e097c..b7d828183 100644 --- a/tests/config.nims +++ b/tests/config.nims @@ -17,3 +17,7 @@ switch("define", "nimLegacyTypeMismatch") hint("Processing", off) # dots can cause annoyances; instead, a single test can test `hintProcessing` + +# uncomment to enable all flaky tests disabled by this flag +# (works through process calls, e.g. tests that invoke nim). +# switch("define", "nimTestsEnableFlaky") diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 63aac0ede..a66177f28 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -7,7 +7,7 @@ discard """ ## A few others could be added to here to simplify code. ## Note: this test is a bit slow but tests a lot of things; please don't disable. -import std/[strformat,os,osproc,unittest] +import std/[strformat,os,osproc,unittest,compilesettings] from std/sequtils import toSeq,mapIt from std/algorithm import sorted import stdtest/[specialpaths, unittest_light] @@ -25,10 +25,7 @@ const # useful when you want to turn only some hints on, and some common ones off. # pending https://github.com/timotheecour/Nim/issues/453, simplify to: `--hints:off` nim = getCurrentCompilerExe() - mode = - when defined(c): "c" - elif defined(cpp): "cpp" - else: static: doAssert false + mode = querySetting(backend) nimcache = buildDir / "nimcacheTrunner" # instead of `querySetting(nimcacheDir)`, avoids stomping on other parallel tests diff --git a/tests/misc/trunner_special.nim b/tests/misc/trunner_special.nim new file mode 100644 index 000000000..47ba44a29 --- /dev/null +++ b/tests/misc/trunner_special.nim @@ -0,0 +1,31 @@ +discard """ + targets: "c cpp" + joinable: false +""" + +#[ +Runs tests that require special treatment, e.g. because they rely on 3rd party code +or require external networking. + +xxx test all tests/untestable/* here, possibly with adjustments to make running times reasonable +]# + +import std/[strformat,os,unittest,compilesettings] +import stdtest/specialpaths + +const + nim = getCurrentCompilerExe() + mode = querySetting(backend) + +proc runCmd(cmd: string) = + let ret = execShellCmd(cmd) + check ret == 0 # allows more than 1 failure + +proc main = + let options = fmt"-b:{mode} --hints:off" + block: # SSL nimDisableCertificateValidation integration tests + runCmd fmt"{nim} r {options} -d:nimDisableCertificateValidation -d:ssl {testsDir}/untestable/thttpclient_ssl_disabled.nim" + block: # SSL certificate check integration tests + runCmd fmt"{nim} r {options} -d:ssl --threads:on {testsDir}/untestable/thttpclient_ssl_remotenetwork.nim" + +main() diff --git a/tests/untestable/thttpclient_ssl.nim b/tests/untestable/thttpclient_ssl.nim deleted file mode 100644 index 038eba99d..000000000 --- a/tests/untestable/thttpclient_ssl.nim +++ /dev/null @@ -1,207 +0,0 @@ -# -# -# Nim - SSL integration tests -# (c) Copyright 2017 Nim contributors -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# -## Warning: this test performs external networking. -## Test with: -## ./bin/nim c -d:ssl -p:. --threads:on -r tests/untestable/thttpclient_ssl.nim -## -## See https://github.com/FedericoCeratto/ssl-comparison/blob/master/README.md -## for a comparison with other clients. - -import - httpclient, - net, - strutils, - threadpool, - unittest - - -type - # bad and dubious tests should not pass SSL validation - # "_broken" mark the test as skipped. Some tests have different - # behavior depending on OS and SSL version! - # TODO: chase and fix the broken tests - Category = enum - good, bad, dubious, good_broken, bad_broken, dubious_broken - CertTest = tuple[url:string, category:Category, desc: string] - -const certificate_tests: array[0..55, CertTest] = [ - ("https://wrong.host.badssl.com/", bad, "wrong.host"), - ("https://captive-portal.badssl.com/", bad, "captive-portal"), - ("https://expired.badssl.com/", bad, "expired"), - ("https://google.com/", good, "good"), - ("https://self-signed.badssl.com/", bad, "self-signed"), - ("https://untrusted-root.badssl.com/", bad, "untrusted-root"), - ("https://revoked.badssl.com/", bad_broken, "revoked"), - ("https://pinning-test.badssl.com/", bad_broken, "pinning-test"), - ("https://no-common-name.badssl.com/", dubious_broken, "no-common-name"), - ("https://no-subject.badssl.com/", dubious_broken, "no-subject"), - ("https://incomplete-chain.badssl.com/", dubious_broken, "incomplete-chain"), - ("https://sha1-intermediate.badssl.com/", bad, "sha1-intermediate"), - ("https://sha256.badssl.com/", good, "sha256"), - ("https://sha384.badssl.com/", good, "sha384"), - ("https://sha512.badssl.com/", good, "sha512"), - ("https://1000-sans.badssl.com/", good, "1000-sans"), - ("https://10000-sans.badssl.com/", good_broken, "10000-sans"), - ("https://ecc256.badssl.com/", good, "ecc256"), - ("https://ecc384.badssl.com/", good, "ecc384"), - ("https://rsa2048.badssl.com/", good, "rsa2048"), - ("https://rsa8192.badssl.com/", dubious_broken, "rsa8192"), - ("http://http.badssl.com/", good, "regular http"), - ("https://http.badssl.com/", bad_broken, "http on https URL"), # FIXME - ("https://cbc.badssl.com/", dubious, "cbc"), - ("https://rc4-md5.badssl.com/", bad, "rc4-md5"), - ("https://rc4.badssl.com/", bad, "rc4"), - ("https://3des.badssl.com/", bad, "3des"), - ("https://null.badssl.com/", bad, "null"), - ("https://mozilla-old.badssl.com/", bad_broken, "mozilla-old"), - ("https://mozilla-intermediate.badssl.com/", dubious_broken, "mozilla-intermediate"), - ("https://mozilla-modern.badssl.com/", good, "mozilla-modern"), - ("https://dh480.badssl.com/", bad, "dh480"), - ("https://dh512.badssl.com/", bad, "dh512"), - ("https://dh1024.badssl.com/", dubious_broken, "dh1024"), - ("https://dh2048.badssl.com/", good, "dh2048"), - ("https://dh-small-subgroup.badssl.com/", bad_broken, "dh-small-subgroup"), - ("https://dh-composite.badssl.com/", bad_broken, "dh-composite"), - ("https://static-rsa.badssl.com/", dubious, "static-rsa"), - ("https://tls-v1-0.badssl.com:1010/", dubious, "tls-v1-0"), - ("https://tls-v1-1.badssl.com:1011/", dubious, "tls-v1-1"), - ("https://invalid-expected-sct.badssl.com/", bad, "invalid-expected-sct"), - ("https://hsts.badssl.com/", good, "hsts"), - ("https://upgrade.badssl.com/", good, "upgrade"), - ("https://preloaded-hsts.badssl.com/", good, "preloaded-hsts"), - ("https://subdomain.preloaded-hsts.badssl.com/", bad, "subdomain.preloaded-hsts"), - ("https://https-everywhere.badssl.com/", good, "https-everywhere"), - ("https://long-extended-subdomain-name-containing-many-letters-and-dashes.badssl.com/", good, - "long-extended-subdomain-name-containing-many-letters-and-dashes"), - ("https://longextendedsubdomainnamewithoutdashesinordertotestwordwrapping.badssl.com/", good, - "longextendedsubdomainnamewithoutdashesinordertotestwordwrapping"), - ("https://superfish.badssl.com/", bad, "(Lenovo) Superfish"), - ("https://edellroot.badssl.com/", bad, "(Dell) eDellRoot"), - ("https://dsdtestprovider.badssl.com/", bad, "(Dell) DSD Test Provider"), - ("https://preact-cli.badssl.com/", bad, "preact-cli"), - ("https://webpack-dev-server.badssl.com/", bad, "webpack-dev-server"), - ("https://mitm-software.badssl.com/", bad, "mitm-software"), - ("https://sha1-2016.badssl.com/", dubious, "sha1-2016"), - ("https://sha1-2017.badssl.com/", bad, "sha1-2017"), -] - - -template evaluate(exception_msg: string, category: Category, desc: string) = - # Evaluate test outcome. Testes flagged as _broken are evaluated and skipped - let raised = (exception_msg.len > 0) - let should_not_raise = category in {good, dubious_broken, bad_broken} - if should_not_raise xor raised: - # we are seeing a known behavior - if category in {good_broken, dubious_broken, bad_broken}: - skip() - if raised: - # check exception_msg == "No SSL certificate found." or - doAssert exception_msg == "No SSL certificate found." or - exception_msg == "SSL Certificate check failed." or - exception_msg.contains("certificate verify failed") or - exception_msg.contains("key too small") or - exception_msg.contains("alert handshake failure") or - exception_msg.contains("bad dh p length") or - # TODO: This one should only triggers for 10000-sans - exception_msg.contains("excessive message size"), exception_msg - - else: - # this is unexpected - if raised: - echo " $# ($#) raised: $#" % [desc, $category, exception_msg] - else: - echo " $# ($#) did not raise" % [desc, $category] - if category in {good, dubious, bad}: - fail() - - -suite "SSL certificate check - httpclient": - - for i, ct in certificate_tests: - - test ct.desc: - var ctx = newContext(verifyMode=CVerifyPeer) - var client = newHttpClient(sslContext=ctx) - let exception_msg = - try: - let a = $client.getContent(ct.url) - "" - except: - getCurrentExceptionMsg() - - evaluate(exception_msg, ct.category, ct.desc) - - - -# threaded tests - - -type - TTOutcome = ref object - desc, exception_msg: string - category: Category - -proc run_t_test(ct: CertTest): TTOutcome {.thread.} = - ## Run test in a {.thread.} - return by ref - result = TTOutcome(desc:ct.desc, exception_msg:"", category: ct.category) - try: - var ctx = newContext(verifyMode=CVerifyPeer) - var client = newHttpClient(sslContext=ctx) - let a = $client.getContent(ct.url) - except: - result.exception_msg = getCurrentExceptionMsg() - - -suite "SSL certificate check - httpclient - threaded": - - # Spawn threads before the "test" blocks - var outcomes = newSeq[FlowVar[TTOutcome]](certificate_tests.len) - for i, ct in certificate_tests: - let t = spawn run_t_test(ct) - outcomes[i] = t - - # create "test" blocks and handle thread outputs - for t in outcomes: - let outcome = ^t # wait for a thread to terminate - - test outcome.desc: - - evaluate(outcome.exception_msg, outcome.category, outcome.desc) - - -# net tests - - -type NetSocketTest = tuple[hostname: string, port: Port, category:Category, desc: string] -const net_tests:array[0..3, NetSocketTest] = [ - ("imap.gmail.com", 993.Port, good, "IMAP"), - ("wrong.host.badssl.com", 443.Port, bad, "wrong.host"), - ("captive-portal.badssl.com", 443.Port, bad, "captive-portal"), - ("expired.badssl.com", 443.Port, bad, "expired"), -] -# TODO: ("null.badssl.com", 443.Port, bad_broken, "null"), - - -suite "SSL certificate check - sockets": - - for ct in net_tests: - - test ct.desc: - - var sock = newSocket() - var ctx = newContext() - ctx.wrapSocket(sock) - let exception_msg = - try: - sock.connect(ct.hostname, ct.port) - "" - except: - getCurrentExceptionMsg() - - evaluate(exception_msg, ct.category, ct.desc) diff --git a/tests/untestable/thttpclient_ssl_disabled.nim b/tests/untestable/thttpclient_ssl_disabled.nim index 0f7e95831..b95dad2c6 100644 --- a/tests/untestable/thttpclient_ssl_disabled.nim +++ b/tests/untestable/thttpclient_ssl_disabled.nim @@ -5,36 +5,32 @@ # See the file "copying.txt", included in this # distribution, for details about the copyright. # -## Warning: this test performs external networking. ## Compile and run with: -## ./bin/nim c -d:nimDisableCertificateValidation -d:ssl -r -p:. tests/untestable/thttpclient_ssl_disabled.nim +## nim r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:nimDisableCertificateValidation -d:ssl -p:. tests/untestable/thttpclient_ssl_disabled.nim -import httpclient, - net, - unittest, - os +from stdtest/testutils import enableRemoteNetworking +when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(openbsd)): + import httpclient, net, unittest -from strutils import contains + const expired = "https://expired.badssl.com/" -const expired = "https://expired.badssl.com/" + doAssert defined(nimDisableCertificateValidation) -doAssert defined(nimDisableCertificateValidation) + suite "SSL certificate check - disabled": -suite "SSL certificate check - disabled": + test "httpclient in insecure mode": + var ctx = newContext(verifyMode = CVerifyPeer) + var client = newHttpClient(sslContext = ctx) + let a = $client.getContent(expired) - test "httpclient in insecure mode": - var ctx = newContext(verifyMode = CVerifyPeer) - var client = newHttpClient(sslContext = ctx) - let a = $client.getContent(expired) + test "httpclient in insecure mode": + var ctx = newContext(verifyMode = CVerifyPeerUseEnvVars) + var client = newHttpClient(sslContext = ctx) + let a = $client.getContent(expired) - test "httpclient in insecure mode": - var ctx = newContext(verifyMode = CVerifyPeerUseEnvVars) - var client = newHttpClient(sslContext = ctx) - let a = $client.getContent(expired) - - test "net socket in insecure mode": - var sock = newSocket() - var ctx = newContext(verifyMode = CVerifyPeerUseEnvVars) - ctx.wrapSocket(sock) - sock.connect("expired.badssl.com", 443.Port) - sock.close + test "net socket in insecure mode": + var sock = newSocket() + var ctx = newContext(verifyMode = CVerifyPeerUseEnvVars) + ctx.wrapSocket(sock) + sock.connect("expired.badssl.com", 443.Port) + sock.close diff --git a/tests/untestable/thttpclient_ssl_remotenetwork.nim b/tests/untestable/thttpclient_ssl_remotenetwork.nim new file mode 100644 index 000000000..ed6866d53 --- /dev/null +++ b/tests/untestable/thttpclient_ssl_remotenetwork.nim @@ -0,0 +1,209 @@ +# +# +# Nim - SSL integration tests +# (c) Copyright 2017 Nim contributors +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# +## Test with: +## nim r --putenv:NIM_TESTAMENT_REMOTE_NETWORKING:1 -d:ssl -p:. --threads:on tests/untestable/thttpclient_ssl_remotenetwork.nim +## +## See https://github.com/FedericoCeratto/ssl-comparison/blob/master/README.md +## for a comparison with other clients. + +from stdtest/testutils import enableRemoteNetworking +when enableRemoteNetworking and (defined(nimTestsEnableFlaky) or not defined(windows) and not defined(openbsd) and not defined(i386)): + # Not supported on Windows due to old openssl version + import + httpclient, + net, + strutils, + threadpool, + unittest + + + type + # bad and dubious tests should not pass SSL validation + # "_broken" mark the test as skipped. Some tests have different + # behavior depending on OS and SSL version! + # TODO: chase and fix the broken tests + Category = enum + good, bad, dubious, good_broken, bad_broken, dubious_broken + CertTest = tuple[url:string, category:Category, desc: string] + + const certificate_tests: array[0..55, CertTest] = [ + ("https://wrong.host.badssl.com/", bad, "wrong.host"), + ("https://captive-portal.badssl.com/", bad, "captive-portal"), + ("https://expired.badssl.com/", bad, "expired"), + ("https://google.com/", good, "good"), + ("https://self-signed.badssl.com/", bad, "self-signed"), + ("https://untrusted-root.badssl.com/", bad, "untrusted-root"), + ("https://revoked.badssl.com/", bad_broken, "revoked"), + ("https://pinning-test.badssl.com/", bad_broken, "pinning-test"), + ("https://no-common-name.badssl.com/", dubious_broken, "no-common-name"), + ("https://no-subject.badssl.com/", dubious_broken, "no-subject"), + ("https://incomplete-chain.badssl.com/", dubious_broken, "incomplete-chain"), + ("https://sha1-intermediate.badssl.com/", bad, "sha1-intermediate"), + ("https://sha256.badssl.com/", good, "sha256"), + ("https://sha384.badssl.com/", good, "sha384"), + ("https://sha512.badssl.com/", good, "sha512"), + ("https://1000-sans.badssl.com/", good, "1000-sans"), + ("https://10000-sans.badssl.com/", good_broken, "10000-sans"), + ("https://ecc256.badssl.com/", good, "ecc256"), + ("https://ecc384.badssl.com/", good, "ecc384"), + ("https://rsa2048.badssl.com/", good, "rsa2048"), + ("https://rsa8192.badssl.com/", dubious_broken, "rsa8192"), + ("http://http.badssl.com/", good, "regular http"), + ("https://http.badssl.com/", bad_broken, "http on https URL"), # FIXME + ("https://cbc.badssl.com/", dubious, "cbc"), + ("https://rc4-md5.badssl.com/", bad, "rc4-md5"), + ("https://rc4.badssl.com/", bad, "rc4"), + ("https://3des.badssl.com/", bad, "3des"), + ("https://null.badssl.com/", bad, "null"), + ("https://mozilla-old.badssl.com/", bad_broken, "mozilla-old"), + ("https://mozilla-intermediate.badssl.com/", dubious_broken, "mozilla-intermediate"), + ("https://mozilla-modern.badssl.com/", good, "mozilla-modern"), + ("https://dh480.badssl.com/", bad, "dh480"), + ("https://dh512.badssl.com/", bad, "dh512"), + ("https://dh1024.badssl.com/", dubious_broken, "dh1024"), + ("https://dh2048.badssl.com/", good, "dh2048"), + ("https://dh-small-subgroup.badssl.com/", bad_broken, "dh-small-subgroup"), + ("https://dh-composite.badssl.com/", bad_broken, "dh-composite"), + ("https://static-rsa.badssl.com/", dubious, "static-rsa"), + ("https://tls-v1-0.badssl.com:1010/", dubious, "tls-v1-0"), + ("https://tls-v1-1.badssl.com:1011/", dubious, "tls-v1-1"), + ("https://invalid-expected-sct.badssl.com/", bad, "invalid-expected-sct"), + ("https://hsts.badssl.com/", good, "hsts"), + ("https://upgrade.badssl.com/", good, "upgrade"), + ("https://preloaded-hsts.badssl.com/", good, "preloaded-hsts"), + ("https://subdomain.preloaded-hsts.badssl.com/", bad, "subdomain.preloaded-hsts"), + ("https://https-everywhere.badssl.com/", good, "https-everywhere"), + ("https://long-extended-subdomain-name-containing-many-letters-and-dashes.badssl.com/", good, + "long-extended-subdomain-name-containing-many-letters-and-dashes"), + ("https://longextendedsubdomainnamewithoutdashesinordertotestwordwrapping.badssl.com/", good, + "longextendedsubdomainnamewithoutdashesinordertotestwordwrapping"), + ("https://superfish.badssl.com/", bad, "(Lenovo) Superfish"), + ("https://edellroot.badssl.com/", bad, "(Dell) eDellRoot"), + ("https://dsdtestprovider.badssl.com/", bad, "(Dell) DSD Test Provider"), + ("https://preact-cli.badssl.com/", bad, "preact-cli"), + ("https://webpack-dev-server.badssl.com/", bad, "webpack-dev-server"), + ("https://mitm-software.badssl.com/", bad, "mitm-software"), + ("https://sha1-2016.badssl.com/", dubious, "sha1-2016"), + ("https://sha1-2017.badssl.com/", bad, "sha1-2017"), + ] + + + template evaluate(exception_msg: string, category: Category, desc: string) = + # Evaluate test outcome. Testes flagged as _broken are evaluated and skipped + let raised = (exception_msg.len > 0) + let should_not_raise = category in {good, dubious_broken, bad_broken} + if should_not_raise xor raised: + # we are seeing a known behavior + if category in {good_broken, dubious_broken, bad_broken}: + skip() + if raised: + # check exception_msg == "No SSL certificate found." or + doAssert exception_msg == "No SSL certificate found." or + exception_msg == "SSL Certificate check failed." or + exception_msg.contains("certificate verify failed") or + exception_msg.contains("key too small") or + exception_msg.contains("alert handshake failure") or + exception_msg.contains("bad dh p length") or + # TODO: This one should only triggers for 10000-sans + exception_msg.contains("excessive message size"), exception_msg + + else: + # this is unexpected + if raised: + echo " $# ($#) raised: $#" % [desc, $category, exception_msg] + else: + echo " $# ($#) did not raise" % [desc, $category] + if category in {good, dubious, bad}: + fail() + + + suite "SSL certificate check - httpclient": + + for i, ct in certificate_tests: + + test ct.desc: + var ctx = newContext(verifyMode=CVerifyPeer) + var client = newHttpClient(sslContext=ctx) + let exception_msg = + try: + let a = $client.getContent(ct.url) + "" + except: + getCurrentExceptionMsg() + + evaluate(exception_msg, ct.category, ct.desc) + + + + # threaded tests + + + type + TTOutcome = ref object + desc, exception_msg: string + category: Category + + proc run_t_test(ct: CertTest): TTOutcome {.thread.} = + ## Run test in a {.thread.} - return by ref + result = TTOutcome(desc:ct.desc, exception_msg:"", category: ct.category) + try: + var ctx = newContext(verifyMode=CVerifyPeer) + var client = newHttpClient(sslContext=ctx) + let a = $client.getContent(ct.url) + except: + result.exception_msg = getCurrentExceptionMsg() + + + suite "SSL certificate check - httpclient - threaded": + + # Spawn threads before the "test" blocks + var outcomes = newSeq[FlowVar[TTOutcome]](certificate_tests.len) + for i, ct in certificate_tests: + let t = spawn run_t_test(ct) + outcomes[i] = t + + # create "test" blocks and handle thread outputs + for t in outcomes: + let outcome = ^t # wait for a thread to terminate + + test outcome.desc: + + evaluate(outcome.exception_msg, outcome.category, outcome.desc) + + + # net tests + + + type NetSocketTest = tuple[hostname: string, port: Port, category:Category, desc: string] + const net_tests:array[0..3, NetSocketTest] = [ + ("imap.gmail.com", 993.Port, good, "IMAP"), + ("wrong.host.badssl.com", 443.Port, bad, "wrong.host"), + ("captive-portal.badssl.com", 443.Port, bad, "captive-portal"), + ("expired.badssl.com", 443.Port, bad, "expired"), + ] + # TODO: ("null.badssl.com", 443.Port, bad_broken, "null"), + + + suite "SSL certificate check - sockets": + + for ct in net_tests: + + test ct.desc: + + var sock = newSocket() + var ctx = newContext() + ctx.wrapSocket(sock) + let exception_msg = + try: + sock.connect(ct.hostname, ct.port) + "" + except: + getCurrentExceptionMsg() + + evaluate(exception_msg, ct.category, ct.desc) |