c++ - ./a.out terminated . Garbage output due to smashing of stack . How to remove this error?

Question

#include <iostream>
#include <fstream>
#include <cstring>
using namespace std;

typedef unsigned long int WORD; /* Should be 32-bit = 4 bytes          */
#define w         32             /* word size in bits                 */
#define r         12             /* number of rounds                  */
#define b         16             /* number of bytes in key            */
#define c          4             /* number words in key               */
                                 /* c = max(1,ceil(8*b/w))            */
#define t         26             /* size of table S = 2*(r+1) words   */
WORD S [t],L[c];                        /* expanded key table                */
WORD P = 0xb7e15163, Q = 0x9e3779b9;    /* magic constants             */
/* Rotation operators. x must be unsigned, to get logical right shift*/
#define ROTL(x,y) (((x)<<(y&(w-1))) | ((x)>>(w-(y&(w-1)))))
#define ROTR(x,y) (((x)>>(y&(w-1))) | ((x)<<(w-(y&(w-1)))))
void RC5_ENCRYPT(WORD *pt, WORD *ct) /* 2 WORD input pt/output ct     */
{ WORD i, A=pt[0]+S[0], B=pt[1]+S[1];
  for (i=1; i<=r; i++)
    { A = ROTL(A^B,B)+S[2*i];
       B = ROTL(B^A,A)+S[2*i+1];
    }
  ct [0] = A ;
  ct [1] = B ;
}
void RC5_DECRYPT(WORD *ct, WORD *pt) /* 2 WORD input ct/output pt     */
{ WORD i, B=ct[1], A=ct[ 0];
  for (i=r; i>0; i--)
    { B = ROTR(B-S [2*i+1],A)^A;
       A = ROTR(A-S [2*i],B)^B;
    }
  pt [1] = B-S [1] ;pt [0] = A-S [0];
}
void RC5_SETUP(unsigned char *K) /* secret input key K 0...b-1]       */
{ WORD i, j, k, u=w/8, A, B, L [c];
   /* Initialize L, then S, then mix key into S */
   for (i=b-1,L[c-1]=0; i!=-1; i--) L[i/u] = (L[i/u]<<8)+K[ i];
   for (S [0]=P,i=1; i<t; i++) S [i] = S [i-1]+Q;
   for (A=B=i=j=k=0; k<3*t; k++,i=(i+1)%t,j=(j+1)%c)      /* 3*t > 3*c */
      { A = S[i] = ROTL(S [i]+(A+B),3);
        B = L[j] = ROTL(L[j]+(A+B),(A+B));
      }
}
void printword(WORD A)
{ WORD k;
  for (k=0 ;k<w; k+=8) printf("%c");
}
int main()
{  
   WORD i, j, k,ptext, pt1 [2], pt2 [2], ct [2] = {0,0};
   ifstream in("key1.txt");
   ifstream in1("plt.txt");
   ofstream out1("cpt.txt");

   if(!in)
   { 
        cout << "Cannot open file.
"; 
        return 1; 
   }

if(!in1)
   {
        cout << "Cannot open file.
";
        return 1;
   }



unsigned char key[b];
  in >> key;
  in1 >> pt1[0];
  in1 >> pt1[0];
  if (sizeof(WORD)!=4)
    printf("RC5 error: WORD has %d bytes.
",sizeof(WORD));







      RC5_SETUP(key);
      RC5_ENCRYPT(pt1,ct);


      printf("
   plaintext "); printword(pt1 [0]); printword(pt1 [1]);
      printf(" ---> ciphertext "); printword(ct [0]); printword(ct [1]);
      printf("
");
 RC5_SETUP(key);
 RC5_DECRYPT(ct,pt2);

 out1<<ct[0];
 out1<<ct[1];
 out1 <<"
";
 printf("
   plaintext "); printword(pt1 [0]); printword(pt1 [1]);

 return 0;
}

Let the plt.txt file contain 101 100 let the key be 111

Answer 1

go step by step, use getch() to stop your program, make sure you understand the all lines of code and try ddd.