After a while of working with this program, it keeps stopping after it creates an output file. I am using Visual Basic 2010 and am still a beginner at this. The homework question is this:
Description (Symmetric Encryption):
Encoding
- Ask the user to type some text
 - Ask the user to type a private key within this range [1-255]. Perform the range validity checking.
 - Encrypt the input text using the provided private key, put the cipher text in a file named by the user.
 
Decoding
- Ask the user to specify the file to decode.
 - Load the cipher text from that file and try to descrypt it without assuming the private key is the same one used in encoding.
 - Put all the trial results in a seperated file named by the user.
 - d. Figure out what the most reasonable result (or original plaintext) is.
 
I can figure out how to encrypt the text, but have trouble creating the output file with the libraries located at this textbook address: http://www.kipirvine.com/asm/examples/index.htm
I will include my code below and show through the commented material how many tries I have had at this. The book does not explain very well to me so if I could see what it is trying to say it would be very helpful!
Thanks in advance!
INCLUDE Irvine32.inc
    BUFMAX = 128                    ; maximum buffer size
    KEYMAX = 128                    ; maximum buffer size
    BUFFER_SIZE = 5000
.data
    sPrompt BYTE        "Enter some text message:       ", 0
    keyPrompt   BYTE        "Enter a private key [1-255]:       ", 0
    cFile   BYTE        "Enter a filename for cypher text: ", 0
    sEncrypt    BYTE        "Cypher text                    ", 0
    sDecrypt    BYTE        "Decrypted:                 ", 0
    error   BYTE        "The key must be within 1 - 255!    ", 0
    buffer  BYTE         BUFMAX + 1 DUP(0)
    bufSize DWORD    ?
    keyStr  BYTE         KEYMAX + 1 DUP(0)
    keySize DWORD    ?
    key     DWORD    ?
    filename    BYTE        "newfile.txt                    ", 0
    fileHdl DWORD   ?
    bufFile BYTE        BUFFER_SIZE DUP (?)
.code
main PROC
    call InputTheString             ; input the plain text
    call InputTheKey                ; input the security key
    call CypherFile             ; input a cypher filename
    ;call TranslateBuffer           ; encrypt the buffer
    ;mov edx, OFFSET sEncrypt           ; display encrypted message
    ;call DisplayMessage
    ;call TranslateBuffer           ; decrypt the buffer
    ;mov edx, OFFSET sDecrypt           ; display decrypted message
    ;call DisplayMessage
    exit
main ENDP
InputTheKey PROC
    pushad                      ; save 32-bit registers
LK: mov edx, OFFSET keyPrompt       ; display a prompt
    call WriteString                ; Enter a private key [1-255]
    call Crlf                       ; start a new line
    call ReadInt                    ; read int into system
    mov key, eax                    ; store int into keyStr
    cmp eax, 255                    ; compare newly read int
    ja LC                       ; jump if above 255 to LC
    cmp eax, 1                  ; compare newly read int
    jb LC                       ; jump if below 1 to LC
    jmp LR                      ; if between range jump to LR
LC: mov edx, OFFSET error           ; The key must be within 1 - 255!
    call WriteString                ; Display the error
    call Crlf                       ; start a new line
    loop LK                     ; loop back to enter the security key
LR: popad                       ; restore the registers
    ret
InputTheKey ENDP
CypherFile PROC
    pushad
    mov edx, OFFSET cFile           ; "Enter a filename for cypher text
    call WriteString                ; Enter a name for encrypted file
    call Crlf
    call ReadString             ; Store the filename in eax
    ;mov filename, eax
    mov edx, OFFSET filename
    ;push eax
    ;mov eax, fileHdl
    ;mov edx, OFFSET bufFile
    ;mov ecx, BUFFER_SIZE
    ;mov edx, "C:\outputtext.txt"
    call CreateOutputFile
    ;mov edx, OFFSET filename
    ;mov ecx, SIZEOF filename
    ;push eax
    ;mov eax, bufSize
    call WriteToFile
    pop eax
    ;call CloseFile
    ret
CypherFile ENDP
InputTheString PROC
    pushad                      ; save 32-bit registers
    mov edx, OFFSET sPrompt         ; display a prompt
    call WriteString                ; "Enter some text message"
    call Crlf                       ; start a new line
    mov ecx, BUFMAX             ; maximum character count
    mov edx, OFFSET buffer          ; point to the buffer
    call ReadString             ; input the string
    mov bufSize, eax                ; save the length
    popad
    ret
InputTheString ENDP
    COMMENT !
DisplayMessage PROC
    pushad
    call WriteString
    mov edx, OFFSET buffer          ; display the buffer
    call WriteString
    call Crlf
    call Crlf
    popad
    ret
DisplayMessage ENDP
TranslateBuffer PROC
    pushad
    mov ecx, bufSize                ; loop counter
    mov esi, 0                  ; index 0 in buffer
    mov edi, 0                  ; index 0 in the key
L1:
    mov al, keyStr[edi]             ; get a character from encryption key
    xor buffer[esi], al             ; translate a byte
    inc esi                     ; point to next byte
    inc edi                     ; go to next position in key
    cmp edi, keySize                ; compare if equal to size of the key
    jb L2
    mov edi, 0                  ; reset to beginning of the key
L2: loop L1
    popad
    ret
TranslateBuffer ENDP
!
END main