aboutsummaryrefslogtreecommitdiff
path: root/src/third_party/libdisasm/ia32_invariant.c
blob: 68ec153d27dd6814a6570aec76c15fe786d1a890 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
#include <stdlib.h>
#include <string.h>

#include "ia32_invariant.h"
#include "ia32_insn.h"
#include "ia32_settings.h"

extern ia32_table_desc_t *ia32_tables;
extern ia32_settings_t ia32_settings;

extern size_t ia32_table_lookup( unsigned char *buf, size_t buf_len,
		unsigned int table, ia32_insn_t **raw_insn,
		 unsigned int *prefixes );


/* -------------------------------- ModR/M, SIB */
/* Convenience flags */
#define MODRM_EA  1                     /* ModR/M is an effective addr */
#define MODRM_reg 2                     /* ModR/M is a register */

/* ModR/M flags */
#define MODRM_RM_SIB            0x04    /* R/M == 100 */
#define MODRM_RM_NOREG          0x05    /* R/B == 101 */
/* if (MODRM.MOD_NODISP && MODRM.RM_NOREG) then just disp32 */
#define MODRM_MOD_NODISP        0x00    /* mod == 00 */
#define MODRM_MOD_DISP8         0x01    /* mod == 01 */
#define MODRM_MOD_DISP32        0x02    /* mod == 10 */
#define MODRM_MOD_NOEA          0x03    /* mod == 11 */
/* 16-bit modrm flags */
#define MOD16_MOD_NODISP      0
#define MOD16_MOD_DISP8       1
#define MOD16_MOD_DISP16      2
#define MOD16_MOD_REG         3

#define MOD16_RM_BXSI         0
#define MOD16_RM_BXDI         1
#define MOD16_RM_BPSI         2
#define MOD16_RM_BPDI         3
#define MOD16_RM_SI           4
#define MOD16_RM_DI           5
#define MOD16_RM_BP           6
#define MOD16_RM_BX           7

/* SIB flags */
#define SIB_INDEX_NONE       0x04
#define SIB_BASE_EBP       0x05
#define SIB_SCALE_NOBASE    0x00

/* Convenience struct for modR/M bitfield */
struct modRM_byte {  
   unsigned int mod : 2;
   unsigned int reg : 3;
   unsigned int rm  : 3; 
};

/* Convenience struct for SIB bitfield */
struct SIB_byte {
   unsigned int scale : 2;
   unsigned int index : 3;
   unsigned int base  : 3;
};

#ifdef WIN32
static void byte_decode(unsigned char b, struct modRM_byte *modrm) {
#else
static inline void byte_decode(unsigned char b, struct modRM_byte *modrm) {
#endif
	/* generic bitfield-packing routine */

	modrm->mod = b >> 6;	/* top 2 bits */
	modrm->reg = (b & 56) >> 3;	/* middle 3 bits */
	modrm->rm = b & 7;	/* bottom 3 bits */
}
static int ia32_invariant_modrm( unsigned char *in, unsigned char *out,
				 unsigned int mode_16, x86_invariant_op_t *op) {
	struct modRM_byte modrm;
	struct SIB_byte sib;
	unsigned char *c, *cin;
	unsigned short *s;
	unsigned int *i;
	int size = 0;	/* modrm byte is already counted */


	byte_decode(*in, &modrm);	/* get bitfields */

	out[0] = in[0];	/* save modrm byte */
	cin = &in[1];
	c = &out[1];
	s = (unsigned short *)&out[1];
	i = (unsigned int *)&out[1];

	op->type = op_expression;
	op->flags |= op_pointer;
	if ( ! mode_16 && modrm.rm == MODRM_RM_SIB && 
			      modrm.mod != MODRM_MOD_NOEA ) {
		size ++;
		byte_decode(*cin, (struct modRM_byte *)(void*)&sib);

		out[1] = in[1];	/* save sib byte */
		cin = &in[2];
		c = &out[2];
		s = (unsigned short *)&out[2];
		i = (unsigned int *)&out[2];

		if ( sib.base == SIB_BASE_EBP && ! modrm.mod ) {
			/* disp 32 is variant! */
			memset( i, X86_WILDCARD_BYTE, 4 );
			size += 4;
		}
	}

	if (! modrm.mod && modrm.rm == 101) {
		if ( mode_16 ) {	/* straight RVA in disp */
			memset( s, X86_WILDCARD_BYTE, 2 );
			size += 2;
		} else {
			memset( i, X86_WILDCARD_BYTE, 2 );
			size += 4;
		}
	} else if (modrm.mod && modrm.mod < 3) {
		if (modrm.mod == MODRM_MOD_DISP8) {	 /* offset in disp */
			*c = *cin;	
			size += 1;
		} else if ( mode_16 ) {
			*s = (* ((unsigned short *) cin));
			size += 2;
		} else {
			*i = (*((unsigned int *) cin));
			size += 4;
		}
	} else if ( modrm.mod == 3 ) {
		op->type = op_register;
		op->flags &= ~op_pointer;
	}

	return (size);
}


static int ia32_decode_invariant( unsigned char *buf, size_t buf_len, 
				ia32_insn_t *t, unsigned char *out, 
				unsigned int prefixes, x86_invariant_t *inv) {

	unsigned int addr_size, op_size, mode_16;
	unsigned int op_flags[3] = { t->dest_flag, t->src_flag, t->aux_flag };
	int x, type, bytes = 0, size = 0, modrm = 0;

	/* set addressing mode */
	if (ia32_settings.options & opt_16_bit) {
		op_size = ( prefixes & PREFIX_OP_SIZE ) ? 4 : 2;
		addr_size = ( prefixes & PREFIX_ADDR_SIZE ) ? 4 : 2;
		mode_16 = ( prefixes & PREFIX_ADDR_SIZE ) ? 0 : 1;
	} else {
		op_size = ( prefixes & PREFIX_OP_SIZE ) ? 2 : 4;
		addr_size = ( prefixes & PREFIX_ADDR_SIZE ) ? 2 : 4;
		mode_16 = ( prefixes & PREFIX_ADDR_SIZE ) ? 1 : 0;
	}

	for (x = 0; x < 3; x++) {
		inv->operands[x].access = (enum x86_op_access) 
						OP_PERM(op_flags[x]);
		inv->operands[x].flags = (enum x86_op_flags) 
						(OP_FLAGS(op_flags[x]) >> 12);

		switch (op_flags[x] & OPTYPE_MASK) {
			case OPTYPE_c:
				size = (op_size == 4) ? 2 : 1;
				break;
			case OPTYPE_a: case OPTYPE_v:
				size = (op_size == 4) ? 4 : 2;
				break;
			case OPTYPE_p:
				size = (op_size == 4) ? 6 : 4;
				break;
			case OPTYPE_b:
				size = 1;
				break;
			case OPTYPE_w:
				size = 2;
				break;
			case OPTYPE_d: case OPTYPE_fs: case OPTYPE_fd:
			case OPTYPE_fe: case OPTYPE_fb: case OPTYPE_fv:
			case OPTYPE_si: case OPTYPE_fx:
				size = 4;
				break;
			case OPTYPE_s:
				size = 6;
				break;
			case OPTYPE_q: case OPTYPE_pi:
				size = 8;
				break;
			case OPTYPE_dq: case OPTYPE_ps: case OPTYPE_ss:
			case OPTYPE_pd: case OPTYPE_sd:
				size = 16;
				break;
			case OPTYPE_m:	
				size = (addr_size == 4) ? 4 : 2;
				break;
			default:
				break;
		}

		type = op_flags[x] & ADDRMETH_MASK;
		switch (type) {
			case ADDRMETH_E: case ADDRMETH_M: case ADDRMETH_Q:
			case ADDRMETH_R: case ADDRMETH_W:
				modrm = 1;	
				bytes += ia32_invariant_modrm( buf, out, 
						mode_16, &inv->operands[x]);
				break;
			case ADDRMETH_C: case ADDRMETH_D: case ADDRMETH_G:
			case ADDRMETH_P: case ADDRMETH_S: case ADDRMETH_T:
			case ADDRMETH_V:
				inv->operands[x].type = op_register;
				modrm = 1;
				break;
			case ADDRMETH_A: case ADDRMETH_O:
				/* pad with xF4's */
				memset( &out[bytes + modrm], X86_WILDCARD_BYTE, 
					size );
				bytes += size;
				inv->operands[x].type = op_offset;
				if ( type == ADDRMETH_O ) {
					inv->operands[x].flags |= op_signed |
								  op_pointer;
				}
				break;
			case ADDRMETH_I: case ADDRMETH_J:
				/* grab imm value */
				if ((op_flags[x] & OPTYPE_MASK) == OPTYPE_v) {
					/* assume this is an address */
					memset( &out[bytes + modrm], 
						X86_WILDCARD_BYTE, size );
				} else {
					memcpy( &out[bytes + modrm], 
						&buf[bytes + modrm], size );
				}
					
				bytes += size;
				if ( type == ADDRMETH_J ) {
					if ( size == 1 ) {
						inv->operands[x].type = 
							op_relative_near;
					} else {
						inv->operands[x].type = 
							op_relative_far;
					}
					inv->operands[x].flags |= op_signed;
				} else {
					inv->operands[x].type = op_immediate;
				}
				break;
			case ADDRMETH_F:
				inv->operands[x].type = op_register;
				break;
			case ADDRMETH_X:
				inv->operands[x].flags |= op_signed |
					  op_pointer | op_ds_seg | op_string;
				break;
			case ADDRMETH_Y:
				inv->operands[x].flags |= op_signed |
					  op_pointer | op_es_seg | op_string;
				break;
			case ADDRMETH_RR:	
				inv->operands[x].type = op_register;
				break;
			case ADDRMETH_II:	
				inv->operands[x].type = op_immediate;
				break;
			default:
				inv->operands[x].type = op_unused;
				break;
		}
	}

	return (bytes + modrm);
}

size_t ia32_disasm_invariant( unsigned char * buf, size_t buf_len, 
		x86_invariant_t *inv ) {
	ia32_insn_t *raw_insn = NULL;
	unsigned int prefixes;
	unsigned int type;
	size_t size;
	
	/* Perform recursive table lookup starting with main table (0) */
	size = ia32_table_lookup( buf, buf_len, 0, &raw_insn, &prefixes );
	if ( size == INVALID_INSN || size > buf_len ) {
		/* TODO: set errno */
		return 0;
	}

	/* copy opcode bytes to buffer */
	memcpy( inv->bytes, buf, size );

	/* set mnemonic type and group */
	type = raw_insn->mnem_flag & ~INS_FLAG_MASK;
        inv->group = (enum x86_insn_group) (INS_GROUP(type)) >> 12;
        inv->type = (enum x86_insn_type) INS_TYPE(type);

	/* handle operands */
	size += ia32_decode_invariant( buf + size, buf_len - size, raw_insn, 
					&buf[size - 1], prefixes, inv );

	inv->size = size;

	return size;		/* return size of instruction in bytes */
}

size_t ia32_disasm_size( unsigned char *buf, size_t buf_len ) {
	x86_invariant_t inv = { {0} };
	return( ia32_disasm_invariant( buf, buf_len, &inv ) );
}