More bogo updates, and checking script

This commit is contained in:
Joseph Birr-Pixton 2017-12-18 21:23:31 +00:00
parent f04798a248
commit 032b63bbdc
2 changed files with 118 additions and 90 deletions

33
bogo/check.py Normal file
View File

@ -0,0 +1,33 @@
import re
import json
config = json.load(open('config.json'))
test_error_set = set(config['TestErrorMap'].keys())
all_tests = set()
failing_tests = set()
unimpl_tests = set()
passed_tests = set()
for line in open('out'):
m = re.match('^(PASSED|UNIMPLEMENTED|FAILED) \((.*)\)$', line.strip())
if m:
status, name = m.groups()
if name in test_error_set:
test_error_set.remove(name)
all_tests.add(name)
if status == 'FAILED':
failing_tests.add(name)
elif status == 'UNIMPLEMENTED':
unimpl_tests.add(name)
elif status == 'PASSED':
passed_tests.add(name)
print len(all_tests), 'total tests'
print len(passed_tests), 'passed'
print len(failing_tests), 'tests failing'
print len(unimpl_tests), 'tests not supported'
if test_error_set:
print 'unknown TestErrorMap keys', test_error_set

View File

@ -113,7 +113,7 @@
"EmptyCertificateList": ":NO_CERTS:",
"SendInvalidRecordType": ":GARBAGE:",
"NoSharedCipher": ":HANDSHAKE_FAILURE:",
"NoSharedCipher-TLS13Draft22": ":HANDSHAKE_FAILURE:",
"NoSharedCipher-TLS13": ":HANDSHAKE_FAILURE:",
"InvalidECDHPoint-Client": ":PEER_MISBEHAVIOUR:",
"InvalidECDHPoint-Server": ":PEER_MISBEHAVIOUR:",
"TrailingMessageData-ClientHello": ":BAD_HANDSHAKE_MSG:",
@ -127,27 +127,25 @@
"TrailingMessageData-ServerKeyExchange": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-ClientKeyExchange": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-CertificateStatus": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13Draft22-ClientHello": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13Draft22-ServerHello": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13Draft22-EncryptedExtensions": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13Draft22-CertificateRequest": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13Draft22-ServerCertificate": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13Draft22-ServerCertificateVerify": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13Draft22-ServerFinished": ":DECRYPTION_FAILED_OR_BAD_RECORD_MAC:",
"TrailingMessageData-TLS13Draft22-ClientCertificate": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13Draft22-ClientCertificateVerify": ":BAD_HANDSHAKE_MSG:",
"MissingKeyShare-Client": ":PEER_MISBEHAVIOUR:",
"MissingKeyShare-Server": ":INCOMPATIBLE:",
"EmptyEncryptedExtensions": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13-ClientHello": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13-ServerHello": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13-EncryptedExtensions": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13-CertificateRequest": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13-ServerCertificate": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13-ServerCertificateVerify": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13-ServerFinished": ":DECRYPTION_FAILED_OR_BAD_RECORD_MAC:",
"TrailingMessageData-TLS13-ClientCertificate": ":BAD_HANDSHAKE_MSG:",
"TrailingMessageData-TLS13-ClientCertificateVerify": ":BAD_HANDSHAKE_MSG:",
"MissingKeyShare-Client-TLS13": ":PEER_MISBEHAVIOUR:",
"MissingKeyShare-Server-TLS13": ":INCOMPATIBLE:",
"EmptyEncryptedExtensions-TLS13": ":BAD_HANDSHAKE_MSG:",
"NoSupportedCurves": ":INCOMPATIBLE:",
"BadECDHECurve": ":PEER_MISBEHAVIOUR:",
"VersionTooLow": ":INCOMPATIBLE:",
"CertMismatchRSA": ":PEER_MISBEHAVIOUR:",
"CertMismatchECDSA": ":PEER_MISBEHAVIOUR:",
"UnofferedExtension-Client": ":PEER_MISBEHAVIOUR:",
"ServerHelloBogusCipher": ":PEER_MISBEHAVIOUR:",
"ServerHelloBogusCipher-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ALPNClient-Mismatch-TLS12": ":PEER_MISBEHAVIOUR:",
"ServerHelloBogusCipher-TLS13": ":PEER_MISBEHAVIOUR:",
"ALPNClient-RejectUnknown-TLS12": ":PEER_MISBEHAVIOUR:",
"ALPNClient-EmptyProtocolName-TLS12": ":PEER_MISBEHAVIOUR:",
"ALPNServer-EmptyProtocolName-TLS12": ":PEER_MISBEHAVIOUR:",
"Verify-ServerAuth-SignatureType": ":PEER_MISBEHAVIOUR:",
@ -155,11 +153,10 @@
"ServerAuth-Enforced": ":PEER_MISBEHAVIOUR:",
"UnofferedExtension-Client": ":PEER_MISBEHAVIOUR:",
"UnknownExtension-Client": ":PEER_MISBEHAVIOUR:",
"OversizedSessionId-TLS12": ":BAD_HANDSHAKE_MSG:",
"KeyUpdate-InvalidRequestMode": ":BAD_HANDSHAKE_MSG:",
"ExtraCompressionMethods-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ExtraCompressionMethods-TLS13": ":PEER_MISBEHAVIOUR:",
"NoNullCompression-TLS12": ":INCOMPATIBLE:",
"NoNullCompression-TLS13Draft22": ":INCOMPATIBLE:",
"NoNullCompression-TLS13": ":INCOMPATIBLE:",
"TLS13Draft22-AES128-GCM-server": ":INCOMPATIBLE:",
"TLS13Draft22-AES128-GCM-client": ":PEER_MISBEHAVIOUR:",
"TLS13Draft22-AES256-GCM-server": ":INCOMPATIBLE:",
@ -179,31 +176,31 @@
"TLS12-AEAD-AES128-GCM-SHA256-client": ":PEER_MISBEHAVIOUR:",
"TLS12-AEAD-AES256-GCM-SHA384-server": ":INCOMPATIBLE:",
"TLS12-AEAD-AES256-GCM-SHA384-client": ":PEER_MISBEHAVIOUR:",
"SkipHelloRetryRequest": ":PEER_MISBEHAVIOUR:",
"SkipHelloRetryRequest-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"NoSupportedVersions": ":INCOMPATIBLE:",
"ClientAuth-Verify-RSA-PKCS1-SHA1-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ServerAuth-Verify-RSA-PKCS1-SHA1-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ClientAuth-Verify-RSA-PKCS1-SHA256-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ServerAuth-Verify-RSA-PKCS1-SHA256-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ClientAuth-Verify-RSA-PKCS1-SHA384-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ServerAuth-Verify-RSA-PKCS1-SHA384-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ClientAuth-Verify-RSA-PKCS1-SHA512-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ServerAuth-Verify-RSA-PKCS1-SHA512-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ServerAuth-Sign-RSA-PKCS1-SHA256-TLS13Draft22": ":INCOMPATIBLE:",
"ServerAuth-Sign-RSA-PKCS1-SHA384-TLS13Draft22": ":INCOMPATIBLE:",
"ServerAuth-Sign-RSA-PKCS1-SHA512-TLS13Draft22": ":INCOMPATIBLE:",
"ClientAuth-Sign-RSA-PKCS1-SHA256-TLS13Draft22": ":INCOMPATIBLE:",
"ClientAuth-Sign-RSA-PKCS1-SHA384-TLS13Draft22": ":INCOMPATIBLE:",
"ClientAuth-Sign-RSA-PKCS1-SHA512-TLS13Draft22": ":INCOMPATIBLE:",
"ALPNClient-Mismatch-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ALPNClient-EmptyProtocolName-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ALPNServer-EmptyProtocolName-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ClientAuth-NoFallback-TLS13Draft22": ":INCOMPATIBLE:",
"ServerAuth-NoFallback-TLS13Draft22": ":INCOMPATIBLE:",
"ClientAuth-Enforced-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"ServerAuth-Enforced-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"SecondClientHelloWrongCurve": ":PEER_MISBEHAVIOUR:",
"SecondClientHelloMissingKeyShare": ":INCOMPATIBLE:",
"ClientAuth-Verify-RSA-PKCS1-SHA1-TLS13": ":PEER_MISBEHAVIOUR:",
"ServerAuth-Verify-RSA-PKCS1-SHA1-TLS13": ":PEER_MISBEHAVIOUR:",
"ClientAuth-Verify-RSA-PKCS1-SHA256-TLS13": ":PEER_MISBEHAVIOUR:",
"ServerAuth-Verify-RSA-PKCS1-SHA256-TLS13": ":PEER_MISBEHAVIOUR:",
"ClientAuth-Verify-RSA-PKCS1-SHA384-TLS13": ":PEER_MISBEHAVIOUR:",
"ServerAuth-Verify-RSA-PKCS1-SHA384-TLS13": ":PEER_MISBEHAVIOUR:",
"ClientAuth-Verify-RSA-PKCS1-SHA512-TLS13": ":PEER_MISBEHAVIOUR:",
"ServerAuth-Verify-RSA-PKCS1-SHA512-TLS13": ":PEER_MISBEHAVIOUR:",
"ServerAuth-Sign-RSA-PKCS1-SHA256-TLS13": ":INCOMPATIBLE:",
"ServerAuth-Sign-RSA-PKCS1-SHA384-TLS13": ":INCOMPATIBLE:",
"ServerAuth-Sign-RSA-PKCS1-SHA512-TLS13": ":INCOMPATIBLE:",
"ClientAuth-Sign-RSA-PKCS1-SHA256-TLS13": ":INCOMPATIBLE:",
"ClientAuth-Sign-RSA-PKCS1-SHA384-TLS13": ":INCOMPATIBLE:",
"ClientAuth-Sign-RSA-PKCS1-SHA512-TLS13": ":INCOMPATIBLE:",
"ALPNClient-EmptyProtocolName-TLS13": ":PEER_MISBEHAVIOUR:",
"ALPNServer-EmptyProtocolName-TLS13": ":PEER_MISBEHAVIOUR:",
"ALPNClient-RejectUnknown-TLS13": ":PEER_MISBEHAVIOUR:",
"ClientAuth-NoFallback-TLS13": ":INCOMPATIBLE:",
"ServerAuth-NoFallback-TLS13": ":INCOMPATIBLE:",
"ClientAuth-Enforced-TLS13": ":PEER_MISBEHAVIOUR:",
"ServerAuth-Enforced-TLS13": ":PEER_MISBEHAVIOUR:",
"SecondClientHelloWrongCurve-TLS13": ":PEER_MISBEHAVIOUR:",
"SecondClientHelloMissingKeyShare-TLS13": ":INCOMPATIBLE:",
"Resume-Server-BinderWrongLength": ":PEER_MISBEHAVIOUR:",
"Resume-Server-NoPSKBinder": ":PEER_MISBEHAVIOUR:",
"Resume-Server-ExtraPSKBinder": ":PEER_MISBEHAVIOUR:",
@ -211,61 +208,59 @@
"Resume-Server-InvalidPSKBinder": ":PEER_MISBEHAVIOUR:",
"Resume-Server-PSKBinderFirstExtension": ":PEER_MISBEHAVIOUR:",
"Resume-Server-UnofferedCipher": ":PEER_MISBEHAVIOUR:",
"Resume-Server-UnofferedCipher-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"Resume-Client-CipherMismatch-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"Resume-Client-PRFMismatch-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"Resume-Client-Mismatch-TLS12-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"Resume-Client-Mismatch-TLS13Draft22-TLS12": ":PEER_MISBEHAVIOUR:",
"TLS13-EmptyRecords": ":PEER_MISBEHAVIOUR:",
"TLS13-OnlyPadding": ":PEER_MISBEHAVIOUR:",
"NoSupportedCurves-TLS13Draft22": ":INCOMPATIBLE:",
"BadECDHECurve-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"InvalidECDHPoint-Client-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"InvalidECDHPoint-Server-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"TLS13-InvalidPSKIdentity": ":PEER_MISBEHAVIOUR:",
"TLS13-AlwaysSelectPSKIdentity": ":PEER_MISBEHAVIOUR:",
"TLS13-TrailingKeyShareData": ":BAD_HANDSHAKE_MSG:",
"HelloRetryRequestCurveMismatch": ":PEER_MISBEHAVIOUR:",
"HelloRetryRequestVersionMismatch": ":BAD_HANDSHAKE_MSG:",
"HelloRetryRequest-DuplicateCookie": ":PEER_MISBEHAVIOUR:",
"HelloRetryRequest-DuplicateCurve": ":PEER_MISBEHAVIOUR:",
"UnknownUnencryptedExtension-Client-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"UnexpectedUnencryptedExtension-Client-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"UnofferedExtension-Client-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"RenegotiationInfo-Forbidden-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"UnknownExtension-Client-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"TLS13-RequestContextInHandshake": ":BAD_HANDSHAKE_MSG:",
"UnnecessaryHelloRetryRequest": ":PEER_MISBEHAVIOUR:",
"UnknownCurve-HelloRetryRequest": ":PEER_MISBEHAVIOUR:",
"DisabledCurve-HelloRetryRequest": ":PEER_MISBEHAVIOUR:",
"HelloRetryRequest-Empty": ":PEER_MISBEHAVIOUR:",
"HelloRetryRequest-EmptyCookie": ":PEER_MISBEHAVIOUR:",
"HelloRetryRequest-Unknown": ":INCOMPATIBLE:",
"Resume-Server-UnofferedCipher-TLS13": ":PEER_MISBEHAVIOUR:",
"Resume-Client-CipherMismatch-TLS13": ":PEER_MISBEHAVIOUR:",
"Resume-Client-PRFMismatch-TLS13": ":PEER_MISBEHAVIOUR:",
"Resume-Client-Mismatch-TLS12-TLS13": ":PEER_MISBEHAVIOUR:",
"Resume-Client-Mismatch-TLS13-TLS12": ":PEER_MISBEHAVIOUR:",
"NoSupportedCurves-TLS13": ":INCOMPATIBLE:",
"BadECDHECurve-TLS13": ":PEER_MISBEHAVIOUR:",
"InvalidECDHPoint-Client-TLS13": ":PEER_MISBEHAVIOUR:",
"InvalidECDHPoint-Server-TLS13": ":PEER_MISBEHAVIOUR:",
"InvalidPSKIdentity-TLS13": ":PEER_MISBEHAVIOUR:",
"AlwaysSelectPSKIdentity-TLS13": ":PEER_MISBEHAVIOUR:",
"TrailingKeyShareData-TLS13": ":BAD_HANDSHAKE_MSG:",
"HelloRetryRequestCurveMismatch-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"HelloRetryRequestVersionMismatch-TLS13Draft22": ":BAD_HANDSHAKE_MSG:",
"HelloRetryRequest-DuplicateCookie-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"HelloRetryRequest-DuplicateCurve-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"UnknownUnencryptedExtension-Client-TLS13": ":PEER_MISBEHAVIOUR:",
"UnexpectedUnencryptedExtension-Client-TLS13": ":PEER_MISBEHAVIOUR:",
"UnofferedExtension-Client-TLS13": ":PEER_MISBEHAVIOUR:",
"RenegotiationInfo-Forbidden-TLS13": ":PEER_MISBEHAVIOUR:",
"UnknownExtension-Client-TLS13": ":PEER_MISBEHAVIOUR:",
"RequestContextInHandshake-TLS13": ":BAD_HANDSHAKE_MSG:",
"UnnecessaryHelloRetryRequest-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"UnknownCurve-HelloRetryRequest-TLS13": ":PEER_MISBEHAVIOUR:",
"DisabledCurve-HelloRetryRequest-TLS13": ":PEER_MISBEHAVIOUR:",
"HelloRetryRequest-Empty-TLS13": ":PEER_MISBEHAVIOUR:",
"HelloRetryRequest-EmptyCookie-TLS13": ":PEER_MISBEHAVIOUR:",
"HelloRetryRequest-Unknown-TLS13": ":INCOMPATIBLE:",
"ServerBogusVersion": ":BAD_HANDSHAKE_MSG:",
"MinimumVersion-Client-TLS13Draft22-TLS12": ":INCOMPATIBLE:",
"MinimumVersion-Client2-TLS13Draft22-TLS12": ":INCOMPATIBLE:",
"MinimumVersion-Server-TLS13Draft22-TLS12": ":INCOMPATIBLE:",
"MinimumVersion-Server2-TLS13Draft22-TLS12": ":INCOMPATIBLE:",
"DuplicateKeyShares": ":PEER_MISBEHAVIOUR:",
"MinimumVersion-Client-TLS13-TLS12": ":INCOMPATIBLE:",
"MinimumVersion-Client2-TLS13-TLS12": ":INCOMPATIBLE:",
"MinimumVersion-Server-TLS13-TLS12": ":INCOMPATIBLE:",
"MinimumVersion-Server2-TLS13-TLS12": ":INCOMPATIBLE:",
"DuplicateKeyShares-TLS13": ":PEER_MISBEHAVIOUR:",
"PartialEncryptedExtensionsWithServerHello": ":PEER_MISBEHAVIOUR:",
"PartialClientFinishedWithClientHello": ":PEER_MISBEHAVIOUR:",
"PointFormat-EncryptedExtensions-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"Ticket-Forbidden-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"PointFormat-EncryptedExtensions-TLS13": ":PEER_MISBEHAVIOUR:",
"Ticket-Forbidden-TLS13": ":PEER_MISBEHAVIOUR:",
"PointFormat-Server-MissingUncompressed": ":INCOMPATIBLE:",
"NegotiatePSKResumption-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"NegotiatePSKResumption-TLS13": ":PEER_MISBEHAVIOUR:",
"PointFormat-Client-MissingUncompressed": ":PEER_MISBEHAVIOUR:",
"SendUnsolicitedOCSPOnCertificate-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"SendUnsolicitedSCTOnCertificate-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"SendUnknownExtensionOnCertificate-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"SendUnsolicitedOCSPOnCertificate-TLS13": ":PEER_MISBEHAVIOUR:",
"SendUnsolicitedSCTOnCertificate-TLS13": ":PEER_MISBEHAVIOUR:",
"SendUnknownExtensionOnCertificate-TLS13": ":PEER_MISBEHAVIOUR:",
"LargePlaintext": ":PEER_MISBEHAVIOUR:",
"SendDuplicateExtensionsOnCerts-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"SendDuplicateExtensionsOnCerts-TLS13": ":PEER_MISBEHAVIOUR:",
"SignedCertificateTimestampListEmpty-Client-TLS12": ":PEER_MISBEHAVIOUR:",
"SignedCertificateTimestampListEmpty-Client-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"SignedCertificateTimestampListEmpty-Client-TLS13": ":PEER_MISBEHAVIOUR:",
"SignedCertificateTimestampListEmptySCT-Client-TLS12": ":PEER_MISBEHAVIOUR:",
"SignedCertificateTimestampListEmptySCT-Client-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"EMS-Forbidden-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"SignedCertificateTimestampListEmptySCT-Client-TLS13": ":PEER_MISBEHAVIOUR:",
"EMS-Forbidden-TLS13": ":PEER_MISBEHAVIOUR:",
"Unclean-Shutdown": ":CLOSE_WITHOUT_CLOSE_NOTIFY:",
"SendExtensionOnClientCertificate-TLS13Draft22": ":PEER_MISBEHAVIOUR:",
"SendExtensionOnClientCertificate-TLS13": ":PEER_MISBEHAVIOUR:",
"ExtendedMasterSecret-NoToYes-Client": ":PEER_MISBEHAVIOUR:",
"ExtendedMasterSecret-YesToNo-Server": ":PEER_MISBEHAVIOUR:",
"ExtendedMasterSecret-YesToNo-Client": ":PEER_MISBEHAVIOUR:"