More encoding tests

22 files changed, 250 insertions, 50 deletions

diff --git a/test/w3c/c89/robots.txt b/test/w3c/c89/robots.txt
new file mode 100644
index 0000000..fdd7340
--- /dev/null
+++ b/test/w3c/c89/robots.txt
@@ -0,0 +1,103 @@
+#
+# robots.txt for http://www.w3.org/
+#
+# $Id: robots.txt,v 1.74 2016/02/11 20:30:26 gerald Exp $
+#
+
+# For use by search.w3.org
+User-agent: W3C-gsa
+Disallow: /Out-Of-Date
+
+User-agent: W3T_SE
+Disallow: /Out-Of-Date
+
+User-agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT; MS Search 4.0 Robot)
+Disallow: /
+
+# W3C Link checker
+User-agent: W3C-checklink
+Disallow:
+
+# the following settings apply to all bots
+User-agent: *
+# Blogs - WordPress
+# https://codex.wordpress.org/Search_Engine_Optimization_for_WordPress#Robots.txt_Optimization
+Disallow: /*/wp-admin/
+Disallow: /*/wp-includes/
+Disallow: /*/wp-content/plugins/
+Disallow: /*/wp-content/cache/
+Disallow: /*/wp-content/themes/
+Disallow: /blog/*/trackback/
+Disallow: /blog/*/feed/
+Disallow: /blog/*/comments/
+Disallow: /blog/*/category/*/*
+Disallow: /blog/*/*/trackback/
+Disallow: /blog/*/*/feed/
+Disallow: /blog/*/*/comments/
+Disallow: /blog/*/*?
+Disallow: /community/trackback/
+Disallow: /community/feed/
+Disallow: /community/comments/
+Disallow: /community/category/*/*
+Disallow: /community/*/trackback/
+Disallow: /community/*/feed/
+Disallow: /community/*/comments/
+Disallow: /community/*/category/*/*
+Disallow: /community/*?
+Disallow: /Consortium/Offices/trackback/
+Disallow: /Consortium/Offices/feed/
+Disallow: /Consortium/Offices/comments/
+Disallow: /Consortium/Offices/category/*/*
+Disallow: /Consortium/Offices/*/trackback/
+Disallow: /Consortium/Offices/*/feed/
+Disallow: /Consortium/Offices/*/comments/
+Disallow: /Consortium/Offices/*?
+# Wikis - Mediawiki
+# https://www.mediawiki.org/wiki/Manual:Robots.txt
+Disallow: /wiki/index.php?
+Disallow: /wiki/index.php/Help
+Disallow: /wiki/index.php/MediaWiki
+Disallow: /wiki/index.php/Special:
+Disallow: /wiki/index.php/Template
+Disallow: /wiki/skins/
+Disallow: /*/wiki/index.php?
+Disallow: /*/wiki/index.php/Help
+Disallow: /*/wiki/index.php/MediaWiki
+Disallow: /*/wiki/index.php/Special:
+Disallow: /*/wiki/index.php/Template
+# various other access-controlled or expensive areas
+Disallow: /2004/ontaria/basic
+Disallow: /Team/
+Disallow: /Project
+Disallow: /Web
+Disallow: /Systems
+Disallow: /History
+Disallow: /Out-Of-Date
+Disallow: /2002/02/mid
+Disallow: /mid/
+Disallow: /2005/06/blog/
+Disallow: /2004/08/W3CTalks
+Disallow: /2007/11/Talks/search
+Disallow: /People/all/
+Disallow: /RDF/Validator/ARPServlet
+Disallow: /RDF/Validator/rdfval
+Disallow: /2003/03/Translations/byLanguage
+Disallow: /2003/03/Translations/byTechnology
+Disallow: /2005/11/Translations/Query
+Disallow: /2000/06/webdata/xslt
+Disallow: /2000/09/webdata/xslt
+Disallow: /2005/08/online_xslt/xslt
+Disallow: /Bugs/
+Disallow: /Search/Mail/Public/
+Disallow: /2006/02/chartergen
+Disallow: /2004/01/pp-impl
+Disallow: /Consortium/supporters
+Disallow: /2007/08/pyRdfa/
+Disallow: /2012/pyRdfa/extract
+Disallow: /WAI/PF/comments/
+Disallow: /participate/conferences.xml
+Disallow: /scripts/
+Disallow: /2005/01/yacker/
+Disallow: /2005/01/yacker?
+Disallow: /2003/09/nschecker?
+
diff --git a/test/w3c/c89/test-enc-ISO-8859-15.scxml b/test/w3c/c89/test-enc-ISO-8859-1.scxml
index 5fdf91c..ed0e3b3 100644
--- a/test/w3c/c89/test-enc-ISO-8859-15.scxml
+++ b/test/w3c/c89/test-enc-ISO-8859-1.scxml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="ISO-8859-15"?>
+<?xml version="1.0" encoding="ISO-8859-1"?>
 <!-- test that neither if clause executes, so that bat is the only event raised. -->
 <scxml xmlns="http://www.w3.org/2005/07/scxml" xmlns:conf="http://www.w3.org/2005/scxml-conformance" initial="s0" version="1.0" datamodel="c89">
   <datamodel>
diff --git a/test/w3c/c89/test-enc-UTF-16.scxml b/test/w3c/c89/test-enc-UTF-16.scxml
new file mode 100644
index 0000000..b78ac1f
--- /dev/null
+++ b/test/w3c/c89/test-enc-UTF-16.scxml
diff --git a/test/w3c/c89/test446.txt b/test/w3c/c89/test446.txt
new file mode 100644
index 0000000..3a26a2e
--- /dev/null
+++ b/test/w3c/c89/test446.txt
@@ -0,0 +1 @@
+[1,2,3]
\ No newline at end of file
diff --git a/test/w3c/c89/test513.txt b/test/w3c/c89/test513.txt
new file mode 100644
index 0000000..08e9b01
--- /dev/null
+++ b/test/w3c/c89/test513.txt
@@ -0,0 +1,16 @@
+This is a fully manual test.  You send a well formed event to the 'location' URL
+ specified for your SCXML interpreter and check that you get a 200 response code back.  
+ One way of doing this, using wget, is shown below (you can use any event name you
+ want, but you must use '_scxmleventname' to indicate the name of the event):
+
+$ wget \
+--post-data='key1=value1&key2=value2' \
+--header '_scxmleventname: test' \
+<!-- URL of your processor goes here -->
+
+--2014-06-25 17:54:49--  http://epikur.local:8090/925c760f-2093-4054-a24c-d972d75f0dcd/basichttp
+Resolving epikur.local (epikur.local)... 10.211.55.2, 10.37.129.2, 10.0.1.54, ...
+Connecting to epikur.local (epikur.local)|10.211.55.2|:8090... connected.
+HTTP request sent, awaiting response... 200 OK
+Length: 0 [text/html]
+Saving to: ‘basichttp’
\ No newline at end of file
diff --git a/test/w3c/c89/test539.txt b/test/w3c/c89/test539.txt
new file mode 100644
index 0000000..de1b0a1
--- /dev/null
+++ b/test/w3c/c89/test539.txt
@@ -0,0 +1,4 @@
+   <books xmlns="">
+     <book title="title1"/>
+     <book title="title2"/>
+   </books> 
\ No newline at end of file
diff --git a/test/w3c/c89/test540.txt b/test/w3c/c89/test540.txt
new file mode 100644
index 0000000..2191239
--- /dev/null
+++ b/test/w3c/c89/test540.txt
@@ -0,0 +1,3 @@
+123
+4 5
+ 
\ No newline at end of file
diff --git a/test/w3c/c89/test552.txt b/test/w3c/c89/test552.txt
new file mode 100644
index 0000000..d8263ee
--- /dev/null
+++ b/test/w3c/c89/test552.txt
@@ -0,0 +1 @@
+2
\ No newline at end of file
diff --git a/test/w3c/c89/test557.txt b/test/w3c/c89/test557.txt
new file mode 100644
index 0000000..a8e51da
--- /dev/null
+++ b/test/w3c/c89/test557.txt
@@ -0,0 +1,4 @@
+<books xmlns="">
+     <book title="title1"/>
+     <book title="title2"/>
+   </books> 
\ No newline at end of file
diff --git a/test/w3c/c89/test558.txt b/test/w3c/c89/test558.txt
new file mode 100644
index 0000000..bb2bcc7
--- /dev/null
+++ b/test/w3c/c89/test558.txt
@@ -0,0 +1,3 @@
+
+this  is 
+a string 
\ No newline at end of file
diff --git a/test/w3c/convert-tests.sh b/test/w3c/convert-tests.sh
index cb138a2..cdbd872 100755
--- a/test/w3c/convert-tests.sh
+++ b/test/w3c/convert-tests.sh
@@ -84,58 +84,80 @@ done
 
 # make sure substitutions are idempotent!
 
-if [ "$LUA" != "" ]; then
-	# percent needs to be escaped in lua patterns
-	sed -i.orig 's/this%20is%20some%20content/this%%20is%%20some%%20content/g' ./lua/test520.scxml
-
-	# x-www-form-urlencode will not preserve the type and we default to string
-	sed -i.orig 's/Var1==2/tonumber(Var1)==2/g' ./lua/test567.scxml
-  
-	# we can pass test562 even though it it ecmascript specific
-	sed -i.orig 's/datamodel=\"ecmascript\"/datamodel=\"lua\"/g' ./lua/test562.scxml
-  
+if [ "$ECMA" != "" ]; then
+	cp txml/*.txt ecma/
+	find ./ecma -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
+	mv ./ecma/test436.scxml ./null
 fi
 
 if [ "$NAMESPACE" != "" ]; then
 	# unnamespace embedded xml in namespace tests
 	sed -i.orig 's/scxml:book/book/g' ./namespace/test557.scxml
 	sed -i.orig 's/scxml:book/book/g' ./namespace/test561.scxml
+	cp txml/*.txt namespace/
+	find ./namespace -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
+	rm ./namespace/test436.scxml
 fi
 
-cp txml/*.txt ecma/
-cp txml/*.txt namespace/
-cp txml/*.txt xpath/
-cp txml/*.txt promela/
-cp txml/*.txt prolog/
-cp txml/*.txt lua/
-cp txml/*.txt jexl/
+if [ "$XPATH" != "" ]; then
+	cp txml/*.txt xpath/
+	find ./xpath -type f -exec grep -Ili 'datamodel="ecmascript"' {} \; |xargs rm -fv
+	rm ./xpath/test436.scxml
+fi
 
-find ./ecma -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
-find ./namespace -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
-find ./xpath -type f -exec grep -Ili 'datamodel="ecmascript"' {} \; |xargs rm -fv
+if [ "$PROMELA" != "" ]; then
+	cp txml/*.txt promela/
+	find ./promela -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
+	find ./promela -type f -exec grep -Ili 'datamodel="ecmascript"' {} \; |xargs rm -fv
+	find ./promela -type f -exec grep -Ili 'datamodel="null"' {} \; |xargs rm -fv
+	rm ./promela/test436.scxml
+fi
 
-find ./promela -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
-find ./promela -type f -exec grep -Ili 'datamodel="ecmascript"' {} \; |xargs rm -fv
-find ./promela -type f -exec grep -Ili 'datamodel="null"' {} \; |xargs rm -fv
+if [ "$PROLOG" != "" ]; then
+	cp txml/*.txt prolog/
+	find ./prolog -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
+	find ./prolog -type f -exec grep -Ili 'datamodel="ecmascript"' {} \; |xargs rm -fv
+	find ./prolog -type f -exec grep -Ili 'datamodel="null"' {} \; |xargs rm -fv
+	rm ./prolog/test436.scxml
+fi
 
-find ./prolog -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
-find ./prolog -type f -exec grep -Ili 'datamodel="ecmascript"' {} \; |xargs rm -fv
-find ./prolog -type f -exec grep -Ili 'datamodel="null"' {} \; |xargs rm -fv
+if [ "$C89" != "" ]; then
+	cp txml/*.txt c89/
+	find ./c89 -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
+	find ./c89 -type f -exec grep -Ili 'datamodel="ecmascript"' {} \; |xargs rm -fv
+	find ./c89 -type f -exec grep -Ili 'datamodel="null"' {} \; |xargs rm -fv
+	rm ./c89/test436.scxml
+fi
 
-find ./c89 -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
-find ./c89 -type f -exec grep -Ili 'datamodel="ecmascript"' {} \; |xargs rm -fv
-find ./c89 -type f -exec grep -Ili 'datamodel="null"' {} \; |xargs rm -fv
+if [ "$LUA" != "" ]; then
+	# percent needs to be escaped in lua patterns
+	sed -i.orig 's/this%20is%20some%20content/this%%20is%%20some%%20content/g' ./lua/test520.scxml
 
-find ./lua -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
-find ./lua -type f -exec grep -Ili 'datamodel="ecmascript"' {} \; |xargs rm -fv
-find ./lua -type f -exec grep -Ili 'datamodel="null"' {} \; |xargs rm -fv
+	# x-www-form-urlencode will not preserve the type and we default to string
+	sed -i.orig 's/Var1==2/tonumber(Var1)==2/g' ./lua/test567.scxml
+  
+	# we can pass test562 even though it it ecmascript specific
+	sed -i.orig 's/datamodel=\"ecmascript\"/datamodel=\"lua\"/g' ./lua/test562.scxml
+
+	cp txml/*.txt lua/
+	find ./lua -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
+	find ./lua -type f -exec grep -Ili 'datamodel="ecmascript"' {} \; |xargs rm -fv
+	find ./lua -type f -exec grep -Ili 'datamodel="null"' {} \; |xargs rm -fv
+	rm ./lua/test436.scxml
+fi
 
-find ./jexl -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
-find ./jexl -type f -exec grep -Ili 'datamodel="ecmascript"' {} \; |xargs rm -fv
-find ./jexl -type f -exec grep -Ili 'datamodel="null"' {} \; |xargs rm -fv
+if [ "$JEXL" != "" ]; then
+	cp txml/*.txt jexl/
+	find ./jexl -type f -exec grep -Ili 'datamodel="xpath"' {} \; |xargs rm -fv
+	find ./jexl -type f -exec grep -Ili 'datamodel="ecmascript"' {} \; |xargs rm -fv
+	find ./jexl -type f -exec grep -Ili 'datamodel="null"' {} \; |xargs rm -fv
+	rm ./jexl/test436.scxml
+fi
 
 # create other encoding tests from the utf8 one
-for ENC in ISO-8859-15 WINDOWS-1252;
+# https://xerces.apache.org/xerces-c/faq-parse-3.html#faq-16
+
+for ENC in UTF-16 ISO-8859-1 WINDOWS-1252
 do
 	export ENC=$ENC
 	find . -name "test-enc-UTF8.scxml" -exec sh -c 'sed "s/UTF-8/${ENC}/g" {} > $(dirname {})/test-enc-${ENC}.tmp.scxml' \;
@@ -146,15 +168,6 @@ done
 # find . -name "tmp.scxml" -exec rm {} \;
 # find . -name "tmp.scxml.orig" -exec rm {} \;
 
-# test436 is the null datamodel
-mv ./ecma/test436.scxml ./null
-rm ./namespace/test436.scxml
-rm ./xpath/test436.scxml
-rm ./promela/test436.scxml
-rm ./prolog/test436.scxml
-rm ./lua/test436.scxml
-rm ./jexl/test436.scxml
-
 # format all SCXML files
 SCXMLS=`find . -type f -name '*.scxml'`
 for SCXML in $SCXMLS
diff --git a/test/w3c/ecma/test-enc-ISO-8859-15.scxml b/test/w3c/ecma/test-enc-ISO-8859-1.scxml
index 331359d..9b1851b 100644
--- a/test/w3c/ecma/test-enc-ISO-8859-15.scxml
+++ b/test/w3c/ecma/test-enc-ISO-8859-1.scxml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="ISO-8859-15"?>
+<?xml version="1.0" encoding="ISO-8859-1"?>
 <!-- test that neither if clause executes, so that bat is the only event raised. -->
 <scxml xmlns="http://www.w3.org/2005/07/scxml" xmlns:conf="http://www.w3.org/2005/scxml-conformance" initial="s0" version="1.0" datamodel="ecmascript">
   <datamodel>
diff --git a/test/w3c/ecma/test-enc-UTF-16.scxml b/test/w3c/ecma/test-enc-UTF-16.scxml
new file mode 100644
index 0000000..c9d3180
--- /dev/null
+++ b/test/w3c/ecma/test-enc-UTF-16.scxml
diff --git a/test/w3c/jexl/test-enc-ISO-8859-15.scxml b/test/w3c/jexl/test-enc-ISO-8859-1.scxml
index d6b0e26..be8ae35 100644
--- a/test/w3c/jexl/test-enc-ISO-8859-15.scxml
+++ b/test/w3c/jexl/test-enc-ISO-8859-1.scxml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="ISO-8859-15"?>
+<?xml version="1.0" encoding="ISO-8859-1"?>
 <!-- test that neither if clause executes, so that bat is the only event raised. -->
 <scxml xmlns="http://www.w3.org/2005/07/scxml" xmlns:conf="http://www.w3.org/2005/scxml-conformance" initial="s0" version="1.0" datamodel="jexl">
   <datamodel>
diff --git a/test/w3c/jexl/test-enc-UTF-16.scxml b/test/w3c/jexl/test-enc-UTF-16.scxml
new file mode 100644
index 0000000..1e439c0
--- /dev/null
+++ b/test/w3c/jexl/test-enc-UTF-16.scxml
diff --git a/test/w3c/lua/test-enc-ISO-8859-15.scxml b/test/w3c/lua/test-enc-ISO-8859-1.scxml
index b026b95..f5e53a7 100644
--- a/test/w3c/lua/test-enc-ISO-8859-15.scxml
+++ b/test/w3c/lua/test-enc-ISO-8859-1.scxml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="ISO-8859-15"?>
+<?xml version="1.0" encoding="ISO-8859-1"?>
 <!-- test that neither if clause executes, so that bat is the only event raised. -->
 <scxml xmlns="http://www.w3.org/2005/07/scxml" xmlns:conf="http://www.w3.org/2005/scxml-conformance" initial="s0" version="1.0" datamodel="lua">
   <datamodel>
diff --git a/test/w3c/lua/test-enc-UTF-16.scxml b/test/w3c/lua/test-enc-UTF-16.scxml
new file mode 100644
index 0000000..01ea8e6
--- /dev/null
+++ b/test/w3c/lua/test-enc-UTF-16.scxml
diff --git a/test/w3c/namespace/test-enc-ISO-8859-15.scxml b/test/w3c/namespace/test-enc-ISO-8859-1.scxml
index 964cd2f..4b815fa 100644
--- a/test/w3c/namespace/test-enc-ISO-8859-15.scxml
+++ b/test/w3c/namespace/test-enc-ISO-8859-1.scxml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="ISO-8859-15"?>
+<?xml version="1.0" encoding="ISO-8859-1"?>
 <!-- test that neither if clause executes, so that bat is the only event raised. -->
 <scxml:scxml xmlns="http://www.w3.org/2005/07/scxml" xmlns:conf="http://www.w3.org/2005/scxml-conformance" xmlns:scxml="http://www.w3.org/2005/07/scxml" initial="s0" version="1.0" datamodel="ecmascript">
   <scxml:datamodel>
diff --git a/test/w3c/namespace/test-enc-UTF-16.scxml b/test/w3c/namespace/test-enc-UTF-16.scxml
new file mode 100644
index 0000000..6bbcfa7
--- /dev/null
+++ b/test/w3c/namespace/test-enc-UTF-16.scxml
diff --git a/test/w3c/promela/test-enc-ISO-8859-1.scxml b/test/w3c/promela/test-enc-ISO-8859-1.scxml
new file mode 100644
index 0000000..c44f071
--- /dev/null
+++ b/test/w3c/promela/test-enc-ISO-8859-1.scxml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<!-- test that neither if clause executes, so that bat is the only event raised. -->
+<scxml xmlns="http://www.w3.org/2005/07/scxml" xmlns:conf="http://www.w3.org/2005/scxml-conformance" initial="s0" version="1.0" datamodel="promela">
+  <datamodel>
+    <data id="Var1" type="string" expr="'ü'"/>
+    <data id="Var2" type="string" expr="'ü'"/>
+  </datamodel>
+  <state id="s0">
+    <transition cond="Var1==Var2" target="s1"/>
+    <transition target="fail"/>
+  </state>
+  <state id="s1">
+    <transition cond="Var1=='ü'" target="pass"/>
+    <transition target="fail"/>
+  </state>
+  <final xmlns:scxml="http://www.w3.org/2005/07/scxml" id="pass">
+    <onentry>
+      <log label="Outcome" expr="'pass'"/>
+    </onentry>
+  </final>
+  <final xmlns:scxml="http://www.w3.org/2005/07/scxml" id="fail">
+    <onentry>
+      <log label="Outcome" expr="'fail'"/>
+    </onentry>
+  </final>
+</scxml>
diff --git a/test/w3c/promela/test-enc-UTF-16.scxml b/test/w3c/promela/test-enc-UTF-16.scxml
new file mode 100644
index 0000000..d02ab96
--- /dev/null
+++ b/test/w3c/promela/test-enc-UTF-16.scxml
diff --git a/test/w3c/promela/test-enc-WINDOWS-1252.scxml b/test/w3c/promela/test-enc-WINDOWS-1252.scxml
new file mode 100644
index 0000000..10c99f6
--- /dev/null
+++ b/test/w3c/promela/test-enc-WINDOWS-1252.scxml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="WINDOWS-1252"?>
+<!-- test that neither if clause executes, so that bat is the only event raised. -->
+<scxml xmlns="http://www.w3.org/2005/07/scxml" xmlns:conf="http://www.w3.org/2005/scxml-conformance" initial="s0" version="1.0" datamodel="promela">
+  <datamodel>
+    <data id="Var1" type="string" expr="'ü'"/>
+    <data id="Var2" type="string" expr="'ü'"/>
+  </datamodel>
+  <state id="s0">
+    <transition cond="Var1==Var2" target="s1"/>
+    <transition target="fail"/>
+  </state>
+  <state id="s1">
+    <transition cond="Var1=='ü'" target="pass"/>
+    <transition target="fail"/>
+  </state>
+  <final xmlns:scxml="http://www.w3.org/2005/07/scxml" id="pass">
+    <onentry>
+      <log label="Outcome" expr="'pass'"/>
+    </onentry>
+  </final>
+  <final xmlns:scxml="http://www.w3.org/2005/07/scxml" id="fail">
+    <onentry>
+      <log label="Outcome" expr="'fail'"/>
+    </onentry>
+  </final>
+</scxml>