Adding kll file emitter and fixing array position merging

- Added kll file emitter
  * Re-constitutes KLL files from various stages of the compilation process
- Using kll file emitter added a basic assignment expression unit test
  * Added unit test to travis
- Fixed array assignment merging when using position operators
  * Required a special case merge
- Update output templates for kiibohd

.gitignore

@@ -65,3 +65,10 @@ __pycache__/
 ###########
 *.attr
 
+# Misc #
+########
+generatedKeymap.h
+generatedPixelmap.c
+kll_defs.h
+tests/generated
+

.travis.yml

@@ -34,6 +34,7 @@ env:
   # Basic KLL Tests
   - DIR=tests SCRIPT=sanity.bash
   - DIR=tests SCRIPT=syntax.bash
+  - DIR=tests SCRIPT=assignment.bash
 
 # Exclusions
 matrix:

common/emitter.py

@@ -106,6 +106,31 @@ class Emitter:
 		)
 
 
+class FileEmitter:
+	'''
+	KLL File Emitter Class
+
+	Base class for any emitter that wants to output a file.
+	Generally, it is recommended to use the TextEmitter as templates are more readable.
+	'''
+	def __init__( self ):
+		'''
+		FileEmitter Initialization
+		'''
+		self.output_files = []
+
+	def generate( self, output_path ):
+		'''
+		Generate output file
+
+		@param contents: String contents of file
+		@param output_path: Path to output file
+		'''
+		for name, contents in self.output_files:
+			with open( "{0}/{1}".format( output_path, name ), 'w' ) as outputFile:
+				outputFile.write( contents )
+
+
 class TextEmitter:
 	'''
 	KLL Text Emitter Class

common/expression.py

@@ -189,6 +189,38 @@ class AssignmentExpression( Expression ):
 
 		return True
 
+	def merge_array( self, new_expression=None ):
+		'''
+		Merge arrays, used for position assignments
+		Merges unconditionally, make sure this is what you want to do first
+
+		If no additional array is specified, just "cap-off" array.
+		This does a proper array expansion into a python list.
+
+		@param new_expression: AssignmentExpression type array, ignore if None
+		'''
+		# First, check if base expression needs to be capped
+		if self.pos is not None:
+			# Generate a new string array
+			new_value = [""] * self.pos
+
+			# Append the old contents to the list
+			new_value.append( self.value )
+			self.value = new_value
+
+			# Clear pos, to indicate that array has been capped
+			self.pos = None
+
+		# Next, if a new_expression has been specified, merge in
+		if new_expression is not None and new_expression.pos is not None:
+			# Check if we need to extend the list
+			new_size = new_expression.pos + 1 - len( self.value )
+			if new_size > 0:
+				self.value.extend( [""] * new_size )
+
+			# Assign value to array
+			self.value[ new_expression.pos ] = new_expression.value
+
 	def variable( self, name, value ):
 		'''
 		Assign variable assignment parameters to expression
@@ -211,6 +243,16 @@ class AssignmentExpression( Expression ):
 		if self.type == 'Variable':
 			return "{0} = {1};".format( self.name, self.value )
 		elif self.type == 'Array':
+			# Output KLL style array, double quoted elements, space-separated
+			if isinstance( self.value, list ):
+				output = "{0}[] =".format( self.name )
+				for value in self.value:
+					output += ' "{0}"'.format( value )
+				output += ";"
+				return output
+
+			# Single array assignment
+			else:
 				return "{0}[{1}] = {2};".format( self.name, self.pos, self.value )
 
 		return "ASSIGNMENT UNKNOWN"

common/organization.py

@@ -497,6 +497,44 @@ class VariableData( Data ):
 	Variable -> Data
 	Array    -> Data
 	'''
+	def add_expression( self, expression, debug ):
+		'''
+		Add expression to data structure
+
+		May have multiple keys to add for a given expression
+
+		In the case of indexed variables, only replaced the specified index
+
+		@param expression: KLL Expression (fully tokenized and parsed)
+		@param debug:      Enable debug output
+		'''
+		# Lookup unique keys for expression
+		keys = expression.unique_keys()
+
+		# Add/Modify expressions in datastructure
+		for key, uniq_expr in keys:
+			# Check which operation we are trying to do, add or modify
+			if debug[0]:
+				if key in self.data.keys():
+					output = self.debug_output['mod'].format( key )
+				else:
+					output = self.debug_output['add'].format( key )
+				print( debug[1] and output or ansi_escape.sub( '', output ) )
+
+			# Check to see if we need to cap-off the array (a position parameter is given)
+			if uniq_expr.type == 'Array' and uniq_expr.pos is not None:
+				# Modify existing array
+				if key in self.data.keys():
+					self.data[ key ].merge_array( uniq_expr )
+
+				# Add new array
+				else:
+					uniq_expr.merge_array()
+					self.data[ key ] = uniq_expr
+
+			# Otherwise just add/replace expression
+			else:
+				self.data[ key ] = uniq_expr
 
 
 class Organization:

emitters/emitters.py

@@ -21,6 +21,7 @@ KLL Emitters Container Classes
 ### Imports ###
 
 import emitters.kiibohd.kiibohd as kiibohd
+import emitters.kll.kll as kll
 import emitters.none.none as none
 
 
@@ -60,6 +61,7 @@ class Emitters:
 		# Dictionary of Emitters
 		self.emitters = {
 			'kiibohd' : kiibohd.Kiibohd( control ),
+			'kll'     : kll.KLL( control ),
 			'none'    : none.Drop( control )
 		}
 

emitters/kiibohd/kiibohd.py

@@ -253,9 +253,9 @@ class Kiibohd( Emitter, TextEmitter ):
 		self.fill_dict['CapabilitiesList'] = "const Capability CapabilitiesList[] = {\n"
 		self.fill_dict['CapabilitiesIndices'] = "typedef enum CapabilityIndex {\n"
 
-		# Keys are pre-sorted
+		# Sorted by C Function name
 		capabilities = full_context.query( 'NameAssociationExpression', 'Capability' )
-		for dkey, dvalue in sorted( capabilities.data.items() ):
+		for dkey, dvalue in sorted( capabilities.data.items(), key=lambda x: x[1].association.name ):
 			funcName = dvalue.association.name
 			argByteWidth = dvalue.association.total_arg_bytes()
 
@@ -266,6 +266,48 @@ class Kiibohd( Emitter, TextEmitter ):
 
 		self.fill_dict['CapabilitiesList'] += "};"
 		self.fill_dict['CapabilitiesIndices'] += "} CapabilityIndex;"
+
+
+		## TODO MOVE ##
+		## Pixel Buffer Setup ##
+		self.fill_dict['PixelBufferSetup'] = "PixelBuf PixelBuffers[] = {\n"
+
+		# Lookup number of buffers
+		bufsize = len( variables.data[ defines.data['Pixel_Buffer_Size'].name ].value )
+		for index in range( bufsize ):
+			# TODO
+			self.fill_dict['PixelBufferSetup'] += "\tPixelBufElem( {0}, {1}, {2}, {3} ),\n".format(
+				0,
+				#variables.data[ defines.data['Pixel_Buffer_Length'].name ].value[ index ],
+				variables.data[ defines.data['Pixel_Buffer_Width'].name ].value[ index ],
+				variables.data[ defines.data['Pixel_Buffer_Size'].name ].value[ index ],
+				#variables.data[ defines.data['Pixel_Buffer_Buffer'].name ].value[ index ],
+				0,
+			)
+		self.fill_dict['PixelBufferSetup'] += "};"
+		print( self.fill_dict['PixelBufferSetup'] )
+
+
+		## Pixel Mapping ##
+		self.fill_dict['PixelMapping'] = ""
+		# TODO
+
+
+		## Pixel Display Mapping ##
+		self.fill_dict['PixelDisplayMapping'] = "const uint8_t Pixel_DisplayMapping[] = {\n"
+		# TODO
+		self.fill_dict['PixelDisplayMapping'] += "};"
+
+
+		## ScanCode to Pixel Mapping ##
+		self.fill_dict['ScanCodeToPixelMapping'] = "const uint8_t Pixel_ScanCodeToPixel[] = {\n"
+		# TODO
+		self.fill_dict['ScanCodeToPixelMapping'] = "};"
+
+
+		## Animations ##
+		self.fill_dict['Animations'] = ""
+		## TODO MOVE END ##
 		return
 
 

emitters/kll/kll.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+'''
+Re-Emits KLL files after processing. May do simplification.
+'''
+
+# Copyright (C) 2016 by Jacob Alexander
+#
+# This file is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This file is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this file.  If not, see <http://www.gnu.org/licenses/>.
+
+### Imports ###
+
+import os
+
+from common.emitter import Emitter, FileEmitter
+
+
+### Decorators ###
+
+## Print Decorator Variables
+ERROR = '\033[5;1;31mERROR\033[0m:'
+WARNING = '\033[5;1;33mWARNING\033[0m:'
+
+
+
+### Classes ###
+
+class KLL( Emitter, FileEmitter ):
+	'''
+	Re-Emits KLL files, may simplify and re-order expressions.
+	'''
+
+	def __init__( self, control ):
+		'''
+		Emitter initialization
+
+		@param control: ControlStage object, used to access data from other stages
+		'''
+		Emitter.__init__( self, control )
+		FileEmitter.__init__( self )
+
+		# Defaults
+		self.target_dir = "generated"
+
+	def command_line_args( self, args ):
+		'''
+		Group parser for command line arguments
+
+		@param args: Name space of processed arguments
+		'''
+		self.target_dir = args.target_dir
+
+	def command_line_flags( self, parser ):
+		'''
+		Group parser for command line options
+
+		@param parser: argparse setup object
+		'''
+		# Create new option group
+		group = parser.add_argument_group('\033[1mKLL Emitter Configuration\033[0m')
+
+		group.add_argument( '--target-dir', type=str, default=self.target_dir,
+			help="Target directory for generated files.\n"
+			"\033[1mDefault\033[0m: {0}\n".format( self.target_dir )
+		)
+
+	def output( self ):
+		'''
+		Final Stage of Emitter
+
+		Generate KLL files
+		'''
+		# Make sure output directory exists
+		os.makedirs( self.target_dir, exist_ok=True )
+
+		# Output list of files to disk
+		self.generate( self.target_dir )
+
+	def reconstitute_store( self, stores, name, debug=False ):
+		'''
+		Takes a list of organization stores and re-constitutes them into a kll file
+
+		@param stores: List of organization stores
+		@param name:   Filename to call list of stores
+		@param debug:  Debug mode (adds a comment to every line with the store key)
+
+		@return: kll file contents
+		'''
+		output = ""
+
+		for store in stores:
+			for key, value in sorted( store.data.items(), key=lambda x: x[0] ):
+				if debug:
+					output += "{0} # {1}\n".format( value, key )
+				else:
+					output += "{0}\n".format( value )
+
+		self.output_files.append( (name, output) )
+
+	def process( self ):
+		'''
+		Emitter Processing
+
+		Takes KLL datastructures and Analysis results then outputs them individually as kll files
+		'''
+		# Acquire Datastructures
+		early_contexts = self.control.stage('DataOrganizationStage').contexts
+		base_context = self.control.stage('DataFinalizationStage').base_context
+		default_context = self.control.stage('DataFinalizationStage').default_context
+		partial_contexts = self.control.stage('DataFinalizationStage').partial_contexts
+		full_context = self.control.stage('DataFinalizationStage').full_context
+
+		# Re-constitute KLL files using contexts of various stages
+		for key, context in early_contexts.items():
+			self.reconstitute_store( context.organization.stores(), "{0}.kll".format( key ) )
+
+		self.reconstitute_store( base_context.organization.stores(), "base.kll" )
+
+		self.reconstitute_store( default_context.organization.stores(), "default.kll" )
+
+		for index, partial in enumerate( partial_contexts ):
+			self.reconstitute_store( partial.organization.stores(), "partial-{0}.kll".format( index ) )
+
+		self.reconstitute_store( full_context.organization.stores(), "final.kll" )
+

examples/assignment.kll

@@ -1,9 +1,9 @@
-Variable = 1;
-Array[] = a b c "b c" 3;
-Index[5] = "this text" thing; # Single element
-Index[6] = moar;
+myVariable = 1;
+myArray[] = a b c "b c" 3;
+myIndex[5] = "this text" thing; # Single element
+myIndex[6] = moar;
 
-Variable => Variable_define;
-Array => Array_define;
-Index => Index_define;
+myVariable => Variable_define;
+myArray => Array_define;
+myIndex => Index_define;
 

templates/kiibohdPixelmap.c

@@ -53,6 +53,7 @@ PixelBuf Pixel_Buffers[] = {
 
 
 // Pixel Mapping
+<|PixelMapping|>
 //#define Pixel_TotalPixels 128 // TODO Generate
 const PixelElement Pixel_Mapping[] = {
 	// Function Row (1-16)
@@ -219,6 +220,7 @@ const PixelElement Pixel_Mapping[] = {
 // - 0.5 key spacing between the columns, in the case where multiple leds should be in the column, one is shifted to the right
 //#define Pixel_DisplayMapping_Cols 38
 //#define Pixel_DisplayMapping_Rows 6
+<|PixelDisplayMapping|>
 const uint8_t Pixel_DisplayMapping[] = {
  97,  1,  0, 98,  0,  2, 99,  3,  0,  4,100,  5,  0,101,  6,102,  7,  0,  8,103,  9,104,  0, 10,105, 11,  0, 12,106, 13,  0,107, 14,108, 15,  0, 16,109,
 128, 17,  0, 18,  0, 19,  0, 20,  0, 21,  0, 22,  0, 23,  0, 24,  0, 25,  0, 26,  0, 27,  0, 28,  0, 29,  0,  0, 30,  0,  0,  0, 31,  0, 32,  0, 33,110,
@@ -269,6 +271,8 @@ const uint8_t testani_frame2[] = {
 #define RGB_White        255,255,255
 #define RGB_Black        0,0,0
 
+#define RGB_MD_Blue      0x00,0xAE,0xDA
+
 
 const uint8_t rainbow_inter_frame0[] = {
 	Pixel_ModRGBCol(0, Set, RGB_Green),
@@ -313,16 +317,32 @@ const uint8_t *clear_pixels_frames[] = {
 };
 
 
+const uint8_t md_blue_frame0[] = {
+	Pixel_ModRGBCol(0, Set, RGB_MD_Blue),
+	Pixel_ModRGBCol(37, Set, RGB_MD_Blue),
+	0,
+};
+
+
+const uint8_t *md_blue_frames[] = {
+	md_blue_frame0,
+	0,
+};
+
+
 // Index of animations
 //  uint8_t *Pixel_Animations[] = { <animation>_frames, ... }
+<|Animations|>
 const uint8_t **Pixel_Animations[] = {
 	testani_frames,
 	rainbow_inter_frames,
 	clear_pixels_frames,
+	md_blue_frames,
 };
 
 
 // ScanCode to Pixel Mapping
+<|ScanCodeToPixelMapping|>
 const uint8_t Pixel_ScanCodeToPixel[] = {
 	// Function Row (1-16)
 	1,

tests/assignment.bash

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Use example .kll files to check basic kll processing
+# Does a diff comparison with a pre-generated file for validation
+# Jacob Alexander 2016
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Common functions
+source ${SCRIPT_DIR}/common.bash
+
+# Start in kll top-level directory
+cd ${SCRIPT_DIR}/..
+
+# Cleanup previously generated files
+rm -rf ${SCRIPT_DIR}/generated
+
+
+# Args used for each of the tests
+ARGS="--emitter kll --target-dir ${SCRIPT_DIR}/generated"
+FAIL_ARGS="--emitter kll --target-dir ${SCRIPT_DIR}/generated --token-debug --parser-token-debug --operation-organization-display --data-organization-display --data-finalization-display"
+
+# Files to check syntax on
+FILES=(
+	examples/assignment.kll
+)
+
+
+## Tests
+
+
+cmds "./kll" "${ARGS}" "${FAIL_ARGS}" ${FILES[@]}
+cmd diff --color=always ${SCRIPT_DIR}/cmp_assignment/final.kll ${SCRIPT_DIR}/generated/final.kll
+
+
+## Tests complete
+
+
+result
+exit $?
+

tests/cmp_assignment/final.kll

@@ -0,0 +1,6 @@
+myArray <= Array_define;
+myIndex <= Index_define;
+myVariable <= Variable_define;
+myArray[] = "a" "b" "c" "b c" "3";
+myIndex[] = "" "" "" "" "" "this textthing" "moar";
+myVariable = 1;