;; Simple test routine to read and parse the normalization test file
;; and run the individual tests mentioned therein.

(in-package "CL-USER")
(defvar *normalization-test* "target:i18n/tests/NormalizationTest.txt")

(defun parse-norm-line (line)
  (declare (string line))
  (cond ((or (char= (aref line 0) #\@)
	     (char= (aref line 0) #\#))
	 ;; Delete lines starting with # or @
	 nil)
	(t
	 ;; Split the line into columns.  There are 5 columns.  Column 6 is a comment.
	 (let* ((p1 (position #\; line))
		(p2 (position #\; line :start (1+ p1)))
		(p3 (position #\; line :start (1+ p2)))
		(p4 (position #\; line :start (1+ p3)))
		(p5 (position #\; line :start (1+ p4))))
	   (labels
	       ((convert (codes)
		  (let ((result nil))
		    (map nil #'(lambda (x)
				 (multiple-value-bind (hi lo)
				     (lisp::surrogates x)
				   (push hi result)
				   (when lo (push lo result))))
			 codes)
		    (coerce (nreverse result) 'simple-string)))
		(read-values (string start end)
		    (with-input-from-string (s string :start start :end end)
		      (let ((*read-base* 16))
			(convert
			 (loop for x = (read s nil nil) while x collect x))))))
	     (values (read-values line 0 p1)
		     (read-values line (1+ p1) p2)
		     (read-values line (1+ p2) p3)
		     (read-values line (1+ p3) p4)
		     (read-values line (1+ p4) p5)))))))
		   
	 
(defun run-nfd-test (&optional (file *normalization-test*))
  (format t "~&NFD test~%")
  (with-open-file (testfile file :direction :input :external-format :iso8859-1)
    (let ((total 0)
	  (failures 0))
      (loop for line = (read-line testfile nil nil) while line do
	   (multiple-value-bind (c1 c2 c3 c4 c5)
	       (parse-norm-line line)
	     (when c1
	       (incf total)
	       (let ((test1 (lisp::string-to-nfd c1))
		     (test2 (lisp::string-to-nfd c2))
		     (test3 (lisp::string-to-nfd c3))
		     (test4 (lisp::string-to-nfd c4))
		     (test5 (lisp::string-to-nfd c5)))
		 (unless (and (string= c3 test1)
			      (string= c3 test2)
			      (string= c3 test3)
			      (string= c5 test4)
			      (string= c5 test5))
		   (incf failures)
		   (format t "**FAILURE**~%")
		   (format t "line:  ~A~%" line)
		   (format t "c3    = ~{~4,'0X~^ ~}~%" (map 'list #'char-code c3))
		   (format t "test1 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test1))
		   (format t "test2 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test2))
		   (format t "test3 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test3))
		   (format t "c5    = ~{~4,'0X~^ ~}~%" (map 'list #'char-code c5))
		   (format t "test4 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test4))
		   (format t "test5 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test5)))))))
      (format t "~A failures from ~A tests (~A%)~%" failures total
	      (* 100.0 (/ failures total))))))

(defun run-nfkd-test (&optional (file *normalization-test*))
  (format t "~&NFKD test~%")
  (with-open-file (testfile file :direction :input :external-format :utf-8)
    (let ((total 0)
	  (failures 0))
      (loop for line = (read-line testfile nil nil) while line do
	   ;;(format t "line = ~S~%" line)
	   (multiple-value-bind (c1 c2 c3 c4 c5)
	       (parse-norm-line line)
	     (when c1
	       (incf total)
	       (let ((test1 (lisp::string-to-nfkd c1))
		     (test2 (lisp::string-to-nfkd c2))
		     (test3 (lisp::string-to-nfkd c3))
		     (test4 (lisp::string-to-nfkd c4))
		     (test5 (lisp::string-to-nfkd c5)))
		 (unless (and (string= c5 test1)
			      (string= c5 test2)
			      (string= c5 test3)
			      (string= c5 test4)
			      (string= c5 test5))
		   (incf failures)
		   (format t "**FAILURE**~%")
		   (format t "line:  ~A~%" line)
		   (format t "c5    = ~{~4,'0X~^ ~}~%" (map 'list #'char-code c5))
		   (format t "test1 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test1))
		   (format t "test2 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test2))
		   (format t "test3 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test3))
		   (format t "test4 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test4))
		   (format t "test5 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test5)))))))
      (format t "~A failures from ~A tests (~A%)~%" failures total
	      (* 100.0 (/ failures total))))))

(defun run-nfc-test (&optional (file *normalization-test*))
  (format t "~&NFC test~%")
  (with-open-file (testfile file :direction :input :external-format :utf-8)
    (let ((total 0)
	  (failures 0))
      (loop for line = (read-line testfile nil nil) while line do
	   (multiple-value-bind (c1 c2 c3 c4 c5)
	       (parse-norm-line line)
	     (when c1
	       (incf total)
	       (let ((test1 (lisp::string-to-nfc c1))
		     (test2 (lisp::string-to-nfc c2))
		     (test3 (lisp::string-to-nfc c3))
		     (test4 (lisp::string-to-nfc c4))
		     (test5 (lisp::string-to-nfc c5)))
		 (unless (and (string= c2 test1)
			      (string= c2 test2)
			      (string= c2 test3)
			      (string= c4 test4)
			      (string= c4 test5))
		   (incf failures)
		   (format t "**FAILURE**~%")
		   (format t "line:  ~A~%" line)
		   (format t "c2    = ~{~4,'0X~^ ~}~%" (map 'list #'char-code c2))
		   (format t "test1 = ~{~4,'0X~^ ~} (~S -> ~S)~%" (map 'list #'char-code test1) c1 test1)
		   (format t "test2 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test2))
		   (format t "test3 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test3))
		   (format t "c4    = ~{~4,'0X~^ ~}~%" (map 'list #'char-code c4))
		   (format t "test4 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test4))
		   (format t "test5 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test5)))))))
      (format t "~A failures from ~A tests (~A%)~%" failures total
	      (* 100.0 (/ failures total))))))

(defun run-nfkc-test (&optional (file *normalization-test*))
  (format t "~&NFKC test~%")
  (with-open-file (testfile file :direction :input :external-format :utf-8)
    (let ((total 0)
	  (failures 0))
      (loop for line = (read-line testfile nil nil) while line do
	   (multiple-value-bind (c1 c2 c3 c4 c5)
	       (parse-norm-line line)
	     (when c1
	       (incf total)
	       (let ((test1 (lisp::string-to-nfkc c1))
		     (test2 (lisp::string-to-nfkc c2))
		     (test3 (lisp::string-to-nfkc c3))
		     (test4 (lisp::string-to-nfkc c4))
		     (test5 (lisp::string-to-nfkc c5)))
		 (unless (and (string= c4 test1)
			      (string= c4 test2)
			      (string= c4 test3)
			      (string= c4 test4)
			      (string= c4 test5))
		   (incf failures)
		   (format t "**FAILURE**~%")
		   (format t "line:  ~A~%" line)
		   (format t "c4    = ~{~4,'0X~^ ~}~%" (map 'list #'char-code c4))
		   (format t "test1 = ~{~4,'0X~^ ~} (~S -> ~S)~%" (map 'list #'char-code test1) c1 test1)
		   (format t "test2 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test2))
		   (format t "test3 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test3))
		   (format t "test4 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test4))
		   (format t "test5 = ~{~4,'0X~^ ~}~%" (map 'list #'char-code test5)))))))
      (format t "~A failures from ~A tests (~A%)~%" failures total
	      (* 100.0 (/ failures total))))))

(defun run-normalization-tests (&optional (file *normalization-test*))
  (run-nfd-test file)
  (run-nfkd-test file)
  (run-nfc-test file)
  (run-nfkc-test file))
