Context Navigation

testbed.cpp @ 177:ee6efdda9ac6

Revision 177:ee6efdda9ac6, 57.0 KB (checked in by Andreas Schaefer <gentryx@…>, 13 months ago)
testing hard-coded offsets to simplify address generation logic

Line
1	#include <cmath>
2	#include <typeinfo>
3	// #include <CL/cl.h>
4	#include <iostream>
5	#include <emmintrin.h>
6	// #include <pmmintrin.h>
7	#include <sys/time.h>
8	#include <vector>
9	#include <libgeodecomp/misc/grid.h>
10
11	using namespace LibGeoDecomp;
12
13	class Scalar2D
14	{
15	public:
16	static int coefficients()
17	{
18	return 0;
19	}
20
21	inline void step(double src, double dst, int offset, int startX, int endX)
22	{
23	for (int x = startX; x < endX; ++x) {
24	dst[x] = (src[x - offset] + src[x - 1] + src[x] + src[x + 1] + src[x + offset]) * 0.2;
25	}
26	}
27
28	int flops()
29	{
30	return 5;
31	}
32	};
33
34	class VectorizedSSEMelbourneShuffle2D
35	{
36	public:
37	static int coefficients()
38	{
39	return 0;
40	}
41
42	inline void step(double src, double dst, int offset, int startX, int endX)
43	{
44	int x = startX;
45	Scalar2D scalarUpdater;
46
47	if ((x & 1) == 1) {
48	scalarUpdater.step(src, dst, offset, x, x + 1);
49	x += 1;
50	}
51
52	__m128d oneFifth = _mm_set_pd(1.0/3.0, 1.0/3.0);
53	__m128d buff0 = _mm_loadu_pd(src + x - 1);
54	__m128d same0 = _mm_load_pd(src + x + 0);
55
56	int paddedEndX = endX - 7;
57	for (; x < paddedEndX; x += 8) {
58	// load center row
59	__m128d same1 = _mm_load_pd(src + x + 2);
60	__m128d same2 = _mm_load_pd(src + x + 4);
61	__m128d same3 = _mm_load_pd(src + x + 6);
62	__m128d same4 = _mm_load_pd(src + x + 8);
63
64	// shuffle values obtain left/right neighbors
65	__m128d buff1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
66	__m128d buff2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
67	__m128d buff3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
68	__m128d buff4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
69
70	// load top row
71	__m128d temp0 = _mm_load_pd(src - offset + x + 0);
72	__m128d temp1 = _mm_load_pd(src - offset + x + 2);
73	__m128d temp2 = _mm_load_pd(src - offset + x + 4);
74	__m128d temp3 = _mm_load_pd(src - offset + x + 6);
75
76	// add center row with left...
77	same0 = _mm_add_pd(same0, buff0);
78	same1 = _mm_add_pd(same1, buff1);
79	same2 = _mm_add_pd(same2, buff2);
80	same3 = _mm_add_pd(same3, buff3);
81
82	// ...and right neighbors
83	same0 = _mm_add_pd(same0, buff1);
84	same1 = _mm_add_pd(same1, buff2);
85	same2 = _mm_add_pd(same2, buff3);
86	same3 = _mm_add_pd(same3, buff4);
87
88	// load bottom row
89	buff0 = _mm_load_pd(src + offset + x + 0);
90	buff1 = _mm_load_pd(src + offset + x + 2);
91	buff2 = _mm_load_pd(src + offset + x + 4);
92	buff3 = _mm_load_pd(src + offset + x + 6);
93
94	// add top row
95	same0 = _mm_add_pd(same0, temp0);
96	same1 = _mm_add_pd(same1, temp1);
97	same2 = _mm_add_pd(same2, temp2);
98	same3 = _mm_add_pd(same3, temp3);
99
100	// add bottom row
101	same0 = _mm_add_pd(same0, buff0);
102	same1 = _mm_add_pd(same1, buff1);
103	same2 = _mm_add_pd(same2, buff2);
104	same3 = _mm_add_pd(same3, buff3);
105
106	// scale down...
107	same0 = _mm_mul_pd(same0, oneFifth);
108	same1 = _mm_mul_pd(same1, oneFifth);
109	same2 = _mm_mul_pd(same2, oneFifth);
110	same3 = _mm_mul_pd(same3, oneFifth);
111
112	// ...and store
113	_mm_store_pd(dst + 0, same0);
114	_mm_store_pd(dst + 2, same1);
115	_mm_store_pd(dst + 4, same2);
116	_mm_store_pd(dst + 6, same3);
117
118	same0 = same4;
119	buff0 = buff4;
120	}
121
122	scalarUpdater.step(src, dst, offset, x, endX);
123	}
124
125	int flops()
126	{
127	return 5;
128	}
129	};
130
131
132	template<typename UPDATER, int DIM>
133	class Benchmark
134	{
135	public:
136	typedef Grid<double, typename Topologies::Cube<DIM>::Topology> GridType;
137
138
139	void run(Coord<DIM> dim, int repeats)
140	{
141	Coord<DIM> coeffDim = dim;
142	coeffDim.c[DIM - 1] *= UPDATER::coefficients();
143	GridType coeff(coeffDim, 0.1);
144	GridType a(dim, 1.0);
145	GridType b(dim, 1.0);
146
147	GridType *oldGrid = &a;
148	GridType *newGrid = &b;
149
150	UPDATER updater;
151
152	long long tStart = getUTtime();
153	std::vector<double*> coefficients(UPDATER::coefficients());
154
155	for (int t = 0; t < repeats; ++t) {
156	for (int z = 1; z < dim.c[2] - 1; ++z) {
157	for (int y = 1; y < dim.c[1] - 1; ++y) {
158	Coord<DIM> c(0, y, z);
159	Coord<DIM> coeffCoord = c;
160	for (int i = 0; i < UPDATER::coefficients(); ++i) {
161	coefficients[i] = &coeff.at(coeffCoord);
162	coeffCoord.c[DIM - 1] += dim.c[DIM - 1];
163	}
164	updater.step(
165	&coefficients[0],
166	&oldGrid->at(c),
167	&newGrid->at(c),
168	dim.c[0],
169	dim.c[0] * dim.c[1],
170	1,
171	dim.c[0] - 1);
172	// double *source[9] = {
173	// &oldGrid->at(Coord<DIM>(0, y - 1, z + 1)),
174	// &oldGrid->at(Coord<DIM>(0, y - 1, z )),
175	// &oldGrid->at(Coord<DIM>(0, y - 1, z - 1)),
176	// &oldGrid->at(Coord<DIM>(0, y, z + 1)),
177	// &oldGrid->at(Coord<DIM>(0, y, z )),
178	// &oldGrid->at(Coord<DIM>(0, y, z - 1)),
179	// &oldGrid->at(Coord<DIM>(0, y + 1, z + 1)),
180	// &oldGrid->at(Coord<DIM>(0, y + 1, z )),
181	// &oldGrid->at(Coord<DIM>(0, y + 1, z - 1))
182	// };
183	// updater.update(source, &newGrid->at(c), 1, dim.c[0] - 1);
184	}
185	}
186	std::swap(newGrid, oldGrid);
187	}
188
189	long long tEnd = getUTtime();
190	evaluate(dim, repeats, tEnd - tStart);
191	}
192
193	void exercise()
194	{
195	std::cout << "# " << typeid(UPDATER).name() << "\n";
196	int lastDim = 0;
197	for (int i = 4; i <= 4096; i *= 2) {
198	int intermediateSteps = 8;
199	for (int j = 0; j < intermediateSteps; ++j) {
200	int d = i * std::pow(2, j * (1.0 / intermediateSteps));
201	if (d % 2) {
202	d += 1;
203	}
204
205	if (d > lastDim) {
206	lastDim = d;
207	Coord<DIM> dim;
208	dim.c[0] = d;
209	dim.c[1] = 4;
210	dim.c[2] = 4;
211	// for (int i = 0; i < DIM; ++i)
212	// dim.c[i] = d;
213	int repeats = std::max(1, 10000000 / dim.prod());
214	run(dim, repeats);
215	}
216	}
217	}
218	std::cout << "\n";
219	}
220
221	private:
222	long long getUTtime()
223	{
224	timeval t;
225	gettimeofday(&t, 0);
226	return (long long)t.tv_sec * 1000000 + t.tv_usec;
227	}
228
229	void evaluate(Coord<DIM> dim, int repeats, long long uTime)
230	{
231	double seconds = 1.0 * uTime / 1000 / 1000;
232	Coord<DIM> inner = dim;
233	for (int i = 0; i < DIM; ++i)
234	inner.c[i] -= 2;
235	double gflops = 1.0 * UPDATER().flops() * inner.prod() *
236	repeats / 1000 / 1000 / 1000 / seconds;
237	std::cout << dim.x() << " " << gflops << "\n";
238	}
239
240
241	};
242
243	#define TN 0
244	#define T 1
245	#define TS 2
246	#define N 3
247	#define C 4
248	#define S 5
249	#define BN 6
250	#define B 7
251	#define BS 8
252
253	class Jacobi3D
254	{
255	public:
256	static int coefficients()
257	{
258	return 1;
259	}
260
261	inline void update(double *src, double dst, int startX, int endX)
262	{
263	int x = startX;
264
265	if ((x & 1) == 1) {
266	updateScalar(src, dst, x, x + 1);
267	x += 1;
268	}
269
270	__m128d oneSeventh = _mm_set_pd(1.0/7.0, 1.0/7.0);
271	__m128d buff0 = _mm_loadu_pd(src[C] + x - 1);
272	__m128d same0 = _mm_load_pd(src[C] + x + 0);
273
274	int paddedEndX = endX - 7;
275	for (; x < paddedEndX; x += 8) {
276	// load center row
277	__m128d same1 = _mm_load_pd(src[0] + x + 2);
278	__m128d same2 = _mm_load_pd(src[0] + x + 4);
279	__m128d same3 = _mm_load_pd(src[0] + x + 6);
280	__m128d same4 = _mm_load_pd(src[0] + x + 8);
281
282	// shuffle values obtain left/right neighbors
283	__m128d buff1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
284	__m128d buff2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
285	__m128d buff3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
286	__m128d buff4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
287
288	// load top row
289	__m128d temp0 = _mm_load_pd(src[T] + x + 0);
290	__m128d temp1 = _mm_load_pd(src[T] + x + 2);
291	__m128d temp2 = _mm_load_pd(src[T] + x + 4);
292	__m128d temp3 = _mm_load_pd(src[T] + x + 6);
293
294	// add center row with left...
295	same0 = _mm_add_pd(same0, buff0);
296	same1 = _mm_add_pd(same1, buff1);
297	same2 = _mm_add_pd(same2, buff2);
298	same3 = _mm_add_pd(same3, buff3);
299
300	// ...and right neighbors
301	same0 = _mm_add_pd(same0, buff1);
302	same1 = _mm_add_pd(same1, buff2);
303	same2 = _mm_add_pd(same2, buff3);
304	same3 = _mm_add_pd(same3, buff4);
305
306	// load bottom row
307	buff0 = _mm_load_pd(src[B] + x + 0);
308	buff1 = _mm_load_pd(src[B] + x + 2);
309	buff2 = _mm_load_pd(src[B] + x + 4);
310	buff3 = _mm_load_pd(src[B] + x + 6);
311
312	// add top row
313	same0 = _mm_add_pd(same0, temp0);
314	same1 = _mm_add_pd(same1, temp1);
315	same2 = _mm_add_pd(same2, temp2);
316	same3 = _mm_add_pd(same3, temp3);
317
318	// load north row
319	temp0 = _mm_load_pd(src[N] + x + 0);
320	temp1 = _mm_load_pd(src[N] + x + 2);
321	temp2 = _mm_load_pd(src[N] + x + 4);
322	temp3 = _mm_load_pd(src[N] + x + 6);
323
324	// add bottom row
325	same0 = _mm_add_pd(same0, buff0);
326	same1 = _mm_add_pd(same1, buff1);
327	same2 = _mm_add_pd(same2, buff2);
328	same3 = _mm_add_pd(same3, buff3);
329
330	// load south row
331	buff0 = _mm_load_pd(src[S] + x + 0);
332	buff1 = _mm_load_pd(src[S] + x + 2);
333	buff2 = _mm_load_pd(src[S] + x + 4);
334	buff3 = _mm_load_pd(src[S] + x + 6);
335
336	// add north row
337	same0 = _mm_add_pd(same0, temp0);
338	same1 = _mm_add_pd(same1, temp1);
339	same2 = _mm_add_pd(same2, temp2);
340	same3 = _mm_add_pd(same3, temp3);
341
342	// add south row
343	same0 = _mm_add_pd(same0, buff0);
344	same1 = _mm_add_pd(same1, buff1);
345	same2 = _mm_add_pd(same2, buff2);
346	same3 = _mm_add_pd(same3, buff3);
347
348	// scale down...
349	same0 = _mm_mul_pd(same0, oneSeventh);
350	same1 = _mm_mul_pd(same1, oneSeventh);
351	same2 = _mm_mul_pd(same2, oneSeventh);
352	same3 = _mm_mul_pd(same3, oneSeventh);
353
354	// ...and store
355	_mm_store_pd(dst + x + 0, same0);
356	_mm_store_pd(dst + x + 2, same1);
357	_mm_store_pd(dst + x + 4, same2);
358	_mm_store_pd(dst + x + 6, same3);
359
360	same0 = same4;
361	buff0 = buff4;
362	}
363
364	updateScalar(src, dst, x, endX);
365	}
366
367	inline void updateScalar(double src[9], double dst, int startX, int endX)
368	{
369	for (int x = startX; x < endX; ++x) {
370	dst[x] =
371	(src[S][x] +
372	src[T][x] +
373	src[C][x - 1] +
374	src[C][x] +
375	src[C][x + 1] +
376	src[B][x] +
377	src[N][x]) * (1.0 / 7.0);
378	}
379	}
380
381	int flops()
382	{
383	return 8;
384	}
385	};
386
387
388	class Scalar3D
389	{
390	public:
391	static int coefficients()
392	{
393	return 7;
394	}
395
396	inline void step(double coeff[7], double src, double *dst, int offsetY, int offsetZ, int startX, int endX)
397	{
398	for (int x = startX; x < endX; ++x) {
399	dst[x] =
400	coeff[0][x] * src[x - offsetZ] +
401	coeff[1][x] * src[x - offsetY] +
402	coeff[2][x] * src[x - 1] +
403	coeff[3][x] * src[x] +
404	coeff[4][x] * src[x + 1] +
405	coeff[5][x] * src[x + offsetY] +
406	coeff[6][x] * src[x + offsetZ];
407	}
408	}
409
410	int flops()
411	{
412	return 13;
413	}
414	};
415
416	class Vectorized3D
417	{
418	public:
419	static int coefficients()
420	{
421	return 7;
422	}
423
424	inline void step(double *coeff, double src, double *dst, int offsetY, int offsetZ, int startX, int endX)
425	{
426	int x = startX;
427	Scalar3D scalarUpdater;
428
429	if ((x & 1) == 1) {
430	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, x + 1);
431	x += 1;
432	}
433
434	__m128d same0 = _mm_load_pd(src + x + 0);
435	__m128d neig0 = _mm_loadu_pd(src + x + 1);
436
437	int paddedEndX = endX - 7;
438	for (; x < paddedEndX; x += 8) {
439	__m128d same1 = _mm_load_pd(src + x + 2);
440	__m128d same2 = _mm_load_pd(src + x + 4);
441	__m128d same3 = _mm_load_pd(src + x + 6);
442	__m128d same4 = _mm_load_pd(src + x + 8);
443
444	__m128d neig1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
445	__m128d neig2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
446	__m128d neig3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
447	__m128d neig4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
448
449	same0 = _mm_mul_pd(same0, _mm_load_pd(&coeff[3][x + 0]));
450	same1 = _mm_mul_pd(same1, _mm_load_pd(&coeff[3][x + 2]));
451	same2 = _mm_mul_pd(same2, _mm_load_pd(&coeff[3][x + 4]));
452	same3 = _mm_mul_pd(same3, _mm_load_pd(&coeff[3][x + 6]));
453
454	__m128d temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[2][x + 0]));
455	__m128d temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[2][x + 2]));
456	__m128d temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[2][x + 4]));
457	__m128d temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[2][x + 6]));
458
459	same0 = _mm_add_pd(same0, temp1);
460	same1 = _mm_add_pd(same1, temp2);
461	same2 = _mm_add_pd(same2, temp3);
462	same3 = _mm_add_pd(same3, temp4);
463
464	temp1 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[4][x + 0]));
465	temp2 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[4][x + 2]));
466	temp3 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[4][x + 4]));
467	temp4 = _mm_mul_pd(neig4, _mm_load_pd(&coeff[4][x + 6]));
468
469	same0 = _mm_add_pd(same0, temp1);
470	same1 = _mm_add_pd(same1, temp2);
471	same2 = _mm_add_pd(same2, temp3);
472	same3 = _mm_add_pd(same3, temp4);
473
474	neig0 = _mm_load_pd(src + x - offsetZ + 0);
475	neig1 = _mm_load_pd(src + x - offsetZ + 2);
476	neig2 = _mm_load_pd(src + x - offsetZ + 4);
477	neig3 = _mm_load_pd(src + x - offsetZ + 6);
478
479	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][x + 0]));
480	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][x + 2]));
481	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][x + 4]));
482	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][x + 6]));
483
484	same0 = _mm_add_pd(same0, temp1);
485	same1 = _mm_add_pd(same1, temp2);
486	same2 = _mm_add_pd(same2, temp3);
487	same3 = _mm_add_pd(same3, temp4);
488
489	neig0 = _mm_load_pd(src + x - offsetY + 0);
490	neig1 = _mm_load_pd(src + x - offsetY + 2);
491	neig2 = _mm_load_pd(src + x - offsetY + 4);
492	neig3 = _mm_load_pd(src + x - offsetY + 6);
493
494	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[1][x + 0]));
495	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[1][x + 2]));
496	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[1][x + 4]));
497	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[1][x + 6]));
498
499	same0 = _mm_add_pd(same0, temp1);
500	same1 = _mm_add_pd(same1, temp2);
501	same2 = _mm_add_pd(same2, temp3);
502	same3 = _mm_add_pd(same3, temp4);
503
504	neig0 = _mm_load_pd(src + x + offsetY + 0);
505	neig1 = _mm_load_pd(src + x + offsetY + 2);
506	neig2 = _mm_load_pd(src + x + offsetY + 4);
507	neig3 = _mm_load_pd(src + x + offsetY + 6);
508
509	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[5][x + 0]));
510	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[5][x + 2]));
511	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[5][x + 4]));
512	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[5][x + 6]));
513
514	same0 = _mm_add_pd(same0, temp1);
515	same1 = _mm_add_pd(same1, temp2);
516	same2 = _mm_add_pd(same2, temp3);
517	same3 = _mm_add_pd(same3, temp4);
518
519	neig0 = _mm_load_pd(src + x + offsetZ + 0);
520	neig1 = _mm_load_pd(src + x + offsetZ + 2);
521	neig2 = _mm_load_pd(src + x + offsetZ + 4);
522	neig3 = _mm_load_pd(src + x + offsetZ + 6);
523
524	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[6][x + 0]));
525	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[6][x + 2]));
526	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[6][x + 4]));
527	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[6][x + 6]));
528
529	same0 = _mm_add_pd(same0, temp1);
530	same1 = _mm_add_pd(same1, temp2);
531	same2 = _mm_add_pd(same2, temp3);
532	same3 = _mm_add_pd(same3, temp4);
533
534	_mm_store_pd(dst + 0, same0);
535	_mm_store_pd(dst + 2, same1);
536	_mm_store_pd(dst + 4, same2);
537	_mm_store_pd(dst + 6, same3);
538
539	same0 = same4;
540	neig0 = neig4;
541
542	// dst[x] =
543	// coeff[0][x] * src[x - offsetZ] +
544	// coeff[1][x] * src[x - offsetY] +
545	// coeff[2][x] * src[x - 1] +
546	// coeff[3][x] * src[x] +
547	// coeff[4][x] * src[x + 1] +
548	// coeff[5][x] * src[x + offsetY] +
549	// coeff[6][x] * src[x + offsetZ];
550	}
551
552	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, endX);
553	}
554
555	int flops()
556	{
557	return 13;
558	}
559	};
560
561	class ExtendedScalar3D
562	{
563	public:
564	static int coefficients()
565	{
566	return 13;
567	}
568
569	inline void step(double coeff[13], double src, double *dst, int offsetY, int offsetZ, int startX, int endX)
570	{
571	for (int x = startX; x < endX; ++x) {
572	dst[x] =
573	coeff[ 0][x] * src[x - offsetZ - offsetY] +
574	coeff[ 1][x] * src[x - offsetZ] +
575	coeff[ 2][x] * src[x - offsetZ + offsetY] +
576	coeff[ 3][x] * src[x - offsetY] +
577	coeff[ 4][x] * src[x - 1] +
578	coeff[ 5][x] * src[x] +
579	coeff[ 6][x] * src[x + 1] +
580	coeff[ 7][x] * src[x + offsetY] +
581	coeff[ 8][x] * src[x + offsetZ - offsetY] +
582	coeff[ 9][x] * src[x + offsetZ] +
583	coeff[10][x] * src[x + offsetZ + offsetY] +
584	coeff[11][x - offsetZ - offsetY] +
585	coeff[11][x - offsetZ] +
586	coeff[11][x - offsetZ + offsetY] +
587	coeff[11][x - offsetY] +
588	coeff[11][x] +
589	coeff[11][x + offsetY] +
590	coeff[11][x + offsetZ - offsetY] +
591	coeff[11][x + offsetZ] +
592	coeff[11][x + offsetZ + offsetY] +
593	coeff[12][x - offsetZ - offsetY] +
594	coeff[12][x - offsetZ] +
595	coeff[12][x - offsetZ + offsetY] +
596	coeff[12][x - offsetY] +
597	coeff[12][x] +
598	coeff[12][x + offsetY] +
599	coeff[12][x + offsetZ - offsetY] +
600	coeff[12][x + offsetZ] +
601	coeff[12][x + offsetZ + offsetY];
602	}
603	}
604
605	int flops()
606	{
607	return 40;
608	}
609	};
610
611	class ExtendedVectorized3D
612	{
613	public:
614	static int coefficients()
615	{
616	return 13;
617	}
618
619	inline void step(double coeff[13], double src, double *dst, int offsetY, int offsetZ, int startX, int endX)
620	{
621	int x = startX;
622	ExtendedScalar3D scalarUpdater;
623
624	if ((x & 1) == 1) {
625	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, x + 1);
626	x += 1;
627	}
628
629	__m128d same0 = _mm_load_pd(src + x + 0);
630	__m128d neig0 = _mm_loadu_pd(src + x + 1);
631
632	int paddedEndX = endX - 7;
633	for (; x < paddedEndX; x += 8) {
634	__m128d same1 = _mm_load_pd(src + x + 2);
635	__m128d same2 = _mm_load_pd(src + x + 4);
636	__m128d same3 = _mm_load_pd(src + x + 6);
637	__m128d same4 = _mm_load_pd(src + x + 8);
638
639	__m128d neig1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
640	__m128d neig2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
641	__m128d neig3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
642	__m128d neig4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
643
644	same0 = _mm_mul_pd(same0, _mm_load_pd(&coeff[3][x + 0]));
645	same1 = _mm_mul_pd(same1, _mm_load_pd(&coeff[3][x + 2]));
646	same2 = _mm_mul_pd(same2, _mm_load_pd(&coeff[3][x + 4]));
647	same3 = _mm_mul_pd(same3, _mm_load_pd(&coeff[3][x + 6]));
648
649	__m128d temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[2][x + 0]));
650	__m128d temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[2][x + 2]));
651	__m128d temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[2][x + 4]));
652	__m128d temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[2][x + 6]));
653
654	same0 = _mm_add_pd(same0, temp1);
655	same1 = _mm_add_pd(same1, temp2);
656	same2 = _mm_add_pd(same2, temp3);
657	same3 = _mm_add_pd(same3, temp4);
658
659	temp1 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[4][x + 0]));
660	temp2 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[4][x + 2]));
661	temp3 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[4][x + 4]));
662	temp4 = _mm_mul_pd(neig4, _mm_load_pd(&coeff[4][x + 6]));
663
664	same0 = _mm_add_pd(same0, temp1);
665	same1 = _mm_add_pd(same1, temp2);
666	same2 = _mm_add_pd(same2, temp3);
667	same3 = _mm_add_pd(same3, temp4);
668
669	neig0 = _mm_load_pd(src + x - offsetZ + 0);
670	neig1 = _mm_load_pd(src + x - offsetZ + 2);
671	neig2 = _mm_load_pd(src + x - offsetZ + 4);
672	neig3 = _mm_load_pd(src + x - offsetZ + 6);
673
674	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][x + 0]));
675	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][x + 2]));
676	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][x + 4]));
677	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][x + 6]));
678
679	same0 = _mm_add_pd(same0, temp1);
680	same1 = _mm_add_pd(same1, temp2);
681	same2 = _mm_add_pd(same2, temp3);
682	same3 = _mm_add_pd(same3, temp4);
683
684	neig0 = _mm_load_pd(src + x - offsetY + 0);
685	neig1 = _mm_load_pd(src + x - offsetY + 2);
686	neig2 = _mm_load_pd(src + x - offsetY + 4);
687	neig3 = _mm_load_pd(src + x - offsetY + 6);
688
689	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[1][x + 0]));
690	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[1][x + 2]));
691	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[1][x + 4]));
692	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[1][x + 6]));
693
694	same0 = _mm_add_pd(same0, temp1);
695	same1 = _mm_add_pd(same1, temp2);
696	same2 = _mm_add_pd(same2, temp3);
697	same3 = _mm_add_pd(same3, temp4);
698
699	neig0 = _mm_load_pd(src + x + offsetY + 0);
700	neig1 = _mm_load_pd(src + x + offsetY + 2);
701	neig2 = _mm_load_pd(src + x + offsetY + 4);
702	neig3 = _mm_load_pd(src + x + offsetY + 6);
703
704	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[5][x + 0]));
705	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[5][x + 2]));
706	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[5][x + 4]));
707	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[5][x + 6]));
708
709	same0 = _mm_add_pd(same0, temp1);
710	same1 = _mm_add_pd(same1, temp2);
711	same2 = _mm_add_pd(same2, temp3);
712	same3 = _mm_add_pd(same3, temp4);
713
714	//xxxxxxxxxxxxx
715	neig0 = _mm_load_pd(src + x + offsetZ + 0);
716	neig1 = _mm_load_pd(src + x + offsetZ + 2);
717	neig2 = _mm_load_pd(src + x + offsetZ + 4);
718	neig3 = _mm_load_pd(src + x + offsetZ + 6);
719
720	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[6][x + 0]));
721	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[6][x + 2]));
722	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[6][x + 4]));
723	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[6][x + 6]));
724
725	same0 = _mm_add_pd(same0, temp1);
726	same1 = _mm_add_pd(same1, temp2);
727	same2 = _mm_add_pd(same2, temp3);
728	same3 = _mm_add_pd(same3, temp4);
729
730	//xxxxxxxxxxxxx
731	neig0 = _mm_load_pd(src + x - offsetZ - offsetY + 0);
732	neig1 = _mm_load_pd(src + x - offsetZ - offsetY + 2);
733	neig2 = _mm_load_pd(src + x - offsetZ - offsetY + 4);
734	neig3 = _mm_load_pd(src + x - offsetZ - offsetY + 6);
735
736	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[7][x + 0]));
737	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[7][x + 2]));
738	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[7][x + 4]));
739	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[7][x + 6]));
740
741	same0 = _mm_add_pd(same0, temp1);
742	same1 = _mm_add_pd(same1, temp2);
743	same2 = _mm_add_pd(same2, temp3);
744	same3 = _mm_add_pd(same3, temp4);
745
746	//xxxxxxxxxxxxx
747	neig0 = _mm_load_pd(src + x - offsetZ + offsetY + 0);
748	neig1 = _mm_load_pd(src + x - offsetZ + offsetY + 2);
749	neig2 = _mm_load_pd(src + x - offsetZ + offsetY + 4);
750	neig3 = _mm_load_pd(src + x - offsetZ + offsetY + 6);
751
752	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[8][x + 0]));
753	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[8][x + 2]));
754	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[8][x + 4]));
755	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[8][x + 6]));
756
757	same0 = _mm_add_pd(same0, temp1);
758	same1 = _mm_add_pd(same1, temp2);
759	same2 = _mm_add_pd(same2, temp3);
760	same3 = _mm_add_pd(same3, temp4);
761
762	//xxxxxxxxxxxxx
763	neig0 = _mm_load_pd(src + x + offsetZ - offsetY + 0);
764	neig1 = _mm_load_pd(src + x + offsetZ - offsetY + 2);
765	neig2 = _mm_load_pd(src + x + offsetZ - offsetY + 4);
766	neig3 = _mm_load_pd(src + x + offsetZ - offsetY + 6);
767
768	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[9][x + 0]));
769	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[9][x + 2]));
770	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[9][x + 4]));
771	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[9][x + 6]));
772
773	same0 = _mm_add_pd(same0, temp1);
774	same1 = _mm_add_pd(same1, temp2);
775	same2 = _mm_add_pd(same2, temp3);
776	same3 = _mm_add_pd(same3, temp4);
777
778	//xxxxxxxxxxxxx
779	neig0 = _mm_load_pd(src + x + offsetZ + offsetY + 0);
780	neig1 = _mm_load_pd(src + x + offsetZ + offsetY + 2);
781	neig2 = _mm_load_pd(src + x + offsetZ + offsetY + 4);
782	neig3 = _mm_load_pd(src + x + offsetZ + offsetY + 6);
783
784	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[10][x + 0]));
785	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[10][x + 2]));
786	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[10][x + 4]));
787	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[10][x + 6]));
788
789	same0 = _mm_add_pd(same0, temp1);
790	same1 = _mm_add_pd(same1, temp2);
791	same2 = _mm_add_pd(same2, temp3);
792	same3 = _mm_add_pd(same3, temp4);
793
794	//xxxxxxxxxxxxx
795	neig0 = _mm_load_pd(coeff[11] + x - offsetZ - offsetY + 0);
796	neig1 = _mm_load_pd(coeff[11] + x - offsetZ - offsetY + 2);
797	neig2 = _mm_load_pd(coeff[11] + x - offsetZ - offsetY + 4);
798	neig3 = _mm_load_pd(coeff[11] + x - offsetZ - offsetY + 6);
799
800	same0 = _mm_add_pd(same0, neig0);
801	same1 = _mm_add_pd(same1, neig1);
802	same2 = _mm_add_pd(same2, neig2);
803	same3 = _mm_add_pd(same3, neig3);
804
805	//xxxxxxxxxxxxx
806	neig0 = _mm_load_pd(coeff[11] + x - offsetZ + 0);
807	neig1 = _mm_load_pd(coeff[11] + x - offsetZ + 2);
808	neig2 = _mm_load_pd(coeff[11] + x - offsetZ + 4);
809	neig3 = _mm_load_pd(coeff[11] + x - offsetZ + 6);
810
811	same0 = _mm_add_pd(same0, neig0);
812	same1 = _mm_add_pd(same1, neig1);
813	same2 = _mm_add_pd(same2, neig2);
814	same3 = _mm_add_pd(same3, neig3);
815
816	//xxxxxxxxxxxxx
817	neig0 = _mm_load_pd(coeff[11] + x - offsetZ + offsetY + 0);
818	neig1 = _mm_load_pd(coeff[11] + x - offsetZ + offsetY + 2);
819	neig2 = _mm_load_pd(coeff[11] + x - offsetZ + offsetY + 4);
820	neig3 = _mm_load_pd(coeff[11] + x - offsetZ + offsetY + 6);
821
822	same0 = _mm_add_pd(same0, neig0);
823	same1 = _mm_add_pd(same1, neig1);
824	same2 = _mm_add_pd(same2, neig2);
825	same3 = _mm_add_pd(same3, neig3);
826
827	//xxxxxxxxxxxxx
828	neig0 = _mm_load_pd(coeff[11] + x - offsetY + 0);
829	neig1 = _mm_load_pd(coeff[11] + x - offsetY + 2);
830	neig2 = _mm_load_pd(coeff[11] + x - offsetY + 4);
831	neig3 = _mm_load_pd(coeff[11] + x - offsetY + 6);
832
833	same0 = _mm_add_pd(same0, neig0);
834	same1 = _mm_add_pd(same1, neig1);
835	same2 = _mm_add_pd(same2, neig2);
836	same3 = _mm_add_pd(same3, neig3);
837
838	//xxxxxxxxxxxxx
839	neig0 = _mm_load_pd(coeff[11] + offsetY + x + 0);
840	neig1 = _mm_load_pd(coeff[11] + offsetY + x + 2);
841	neig2 = _mm_load_pd(coeff[11] + offsetY + x + 4);
842	neig3 = _mm_load_pd(coeff[11] + offsetY + x + 6);
843
844	same0 = _mm_add_pd(same0, neig0);
845	same1 = _mm_add_pd(same1, neig1);
846	same2 = _mm_add_pd(same2, neig2);
847	same3 = _mm_add_pd(same3, neig3);
848
849	//xxxxxxxxxxxxx
850	neig0 = _mm_load_pd(coeff[11] + x + 0);
851	neig1 = _mm_load_pd(coeff[11] + x + 2);
852	neig2 = _mm_load_pd(coeff[11] + x + 4);
853	neig3 = _mm_load_pd(coeff[11] + x + 6);
854
855	same0 = _mm_add_pd(same0, neig0);
856	same1 = _mm_add_pd(same1, neig1);
857	same2 = _mm_add_pd(same2, neig2);
858	same3 = _mm_add_pd(same3, neig3);
859
860	//xxxxxxxxxxxxx
861	neig0 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 0);
862	neig1 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 2);
863	neig2 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 4);
864	neig3 = _mm_load_pd(coeff[11] + x + offsetZ - offsetY + 6);
865
866	same0 = _mm_add_pd(same0, neig0);
867	same1 = _mm_add_pd(same1, neig1);
868	same2 = _mm_add_pd(same2, neig2);
869	same3 = _mm_add_pd(same3, neig3);
870
871	//xxxxxxxxxxxxx
872	neig0 = _mm_load_pd(coeff[11] + x + offsetZ + 0);
873	neig1 = _mm_load_pd(coeff[11] + x + offsetZ + 2);
874	neig2 = _mm_load_pd(coeff[11] + x + offsetZ + 4);
875	neig3 = _mm_load_pd(coeff[11] + x + offsetZ + 6);
876
877	same0 = _mm_add_pd(same0, neig0);
878	same1 = _mm_add_pd(same1, neig1);
879	same2 = _mm_add_pd(same2, neig2);
880	same3 = _mm_add_pd(same3, neig3);
881
882	//xxxxxxxxxxxxx
883	neig0 = _mm_load_pd(coeff[11] + x + offsetZ + offsetY + 0);
884	neig1 = _mm_load_pd(coeff[11] + x + offsetZ + offsetY + 2);
885	neig2 = _mm_load_pd(coeff[11] + x + offsetZ + offsetY + 4);
886	neig3 = _mm_load_pd(coeff[11] + x + offsetZ + offsetY + 6);
887
888	same0 = _mm_add_pd(same0, neig0);
889	same1 = _mm_add_pd(same1, neig1);
890	same2 = _mm_add_pd(same2, neig2);
891	same3 = _mm_add_pd(same3, neig3);
892
893	//xxxxxxxxxxxxx
894	neig0 = _mm_load_pd(coeff[12] + x - offsetZ - offsetY + 0);
895	neig1 = _mm_load_pd(coeff[12] + x - offsetZ - offsetY + 2);
896	neig2 = _mm_load_pd(coeff[12] + x - offsetZ - offsetY + 4);
897	neig3 = _mm_load_pd(coeff[12] + x - offsetZ - offsetY + 6);
898
899	same0 = _mm_add_pd(same0, neig0);
900	same1 = _mm_add_pd(same1, neig1);
901	same2 = _mm_add_pd(same2, neig2);
902	same3 = _mm_add_pd(same3, neig3);
903
904	//xxxxxxxxxxxxx
905	neig0 = _mm_load_pd(coeff[12] + x - offsetZ + 0);
906	neig1 = _mm_load_pd(coeff[12] + x - offsetZ + 2);
907	neig2 = _mm_load_pd(coeff[12] + x - offsetZ + 4);
908	neig3 = _mm_load_pd(coeff[12] + x - offsetZ + 6);
909
910	same0 = _mm_add_pd(same0, neig0);
911	same1 = _mm_add_pd(same1, neig1);
912	same2 = _mm_add_pd(same2, neig2);
913	same3 = _mm_add_pd(same3, neig3);
914
915	//xxxxxxxxxxxxx
916	neig0 = _mm_load_pd(coeff[12] + x - offsetZ + offsetY + 0);
917	neig1 = _mm_load_pd(coeff[12] + x - offsetZ + offsetY + 2);
918	neig2 = _mm_load_pd(coeff[12] + x - offsetZ + offsetY + 4);
919	neig3 = _mm_load_pd(coeff[12] + x - offsetZ + offsetY + 6);
920
921	same0 = _mm_add_pd(same0, neig0);
922	same1 = _mm_add_pd(same1, neig1);
923	same2 = _mm_add_pd(same2, neig2);
924	same3 = _mm_add_pd(same3, neig3);
925
926	//xxxxxxxxxxxxx
927	neig0 = _mm_load_pd(coeff[12] + x - offsetY + 0);
928	neig1 = _mm_load_pd(coeff[12] + x - offsetY + 2);
929	neig2 = _mm_load_pd(coeff[12] + x - offsetY + 4);
930	neig3 = _mm_load_pd(coeff[12] + x - offsetY + 6);
931
932	same0 = _mm_add_pd(same0, neig0);
933	same1 = _mm_add_pd(same1, neig1);
934	same2 = _mm_add_pd(same2, neig2);
935	same3 = _mm_add_pd(same3, neig3);
936
937	//xxxxxxxxxxxxx
938	neig0 = _mm_load_pd(coeff[12] + offsetY + x + 0);
939	neig1 = _mm_load_pd(coeff[12] + offsetY + x + 2);
940	neig2 = _mm_load_pd(coeff[12] + offsetY + x + 4);
941	neig3 = _mm_load_pd(coeff[12] + offsetY + x + 6);
942
943	same0 = _mm_add_pd(same0, neig0);
944	same1 = _mm_add_pd(same1, neig1);
945	same2 = _mm_add_pd(same2, neig2);
946	same3 = _mm_add_pd(same3, neig3);
947
948	//xxxxxxxxxxxxx
949	neig0 = _mm_load_pd(coeff[12] + x + 0);
950	neig1 = _mm_load_pd(coeff[12] + x + 2);
951	neig2 = _mm_load_pd(coeff[12] + x + 4);
952	neig3 = _mm_load_pd(coeff[12] + x + 6);
953
954	same0 = _mm_add_pd(same0, neig0);
955	same1 = _mm_add_pd(same1, neig1);
956	same2 = _mm_add_pd(same2, neig2);
957	same3 = _mm_add_pd(same3, neig3);
958
959	//xxxxxxxxxxxxx
960	neig0 = _mm_load_pd(coeff[12] + x + offsetZ - offsetY + 0);
961	neig1 = _mm_load_pd(coeff[12] + x + offsetZ - offsetY + 2);
962	neig2 = _mm_load_pd(coeff[12] + x + offsetZ - offsetY + 4);
963	neig3 = _mm_load_pd(coeff[12] + x + offsetZ - offsetY + 6);
964
965	same0 = _mm_add_pd(same0, neig0);
966	same1 = _mm_add_pd(same1, neig1);
967	same2 = _mm_add_pd(same2, neig2);
968	same3 = _mm_add_pd(same3, neig3);
969
970	//xxxxxxxxxxxxx
971	neig0 = _mm_load_pd(coeff[12] + x + offsetZ + 0);
972	neig1 = _mm_load_pd(coeff[12] + x + offsetZ + 2);
973	neig2 = _mm_load_pd(coeff[12] + x + offsetZ + 4);
974	neig3 = _mm_load_pd(coeff[12] + x + offsetZ + 6);
975
976	same0 = _mm_add_pd(same0, neig0);
977	same1 = _mm_add_pd(same1, neig1);
978	same2 = _mm_add_pd(same2, neig2);
979	same3 = _mm_add_pd(same3, neig3);
980
981	//xxxxxxxxxxxxx
982	neig0 = _mm_load_pd(coeff[12] + x + offsetZ + offsetY + 0);
983	neig1 = _mm_load_pd(coeff[12] + x + offsetZ + offsetY + 2);
984	neig2 = _mm_load_pd(coeff[12] + x + offsetZ + offsetY + 4);
985	neig3 = _mm_load_pd(coeff[12] + x + offsetZ + offsetY + 6);
986
987	same0 = _mm_add_pd(same0, neig0);
988	same1 = _mm_add_pd(same1, neig1);
989	same2 = _mm_add_pd(same2, neig2);
990	same3 = _mm_add_pd(same3, neig3);
991
992	//yyyyyyyyyyyyy
993	_mm_store_pd(dst + x + 0, same0);
994	_mm_store_pd(dst + x + 2, same1);
995	_mm_store_pd(dst + x + 4, same2);
996	_mm_store_pd(dst + x + 6, same3);
997
998	same0 = same4;
999	neig0 = neig4;
1000
1001	// dst[x] =
1002	// coeff[0][x] * src[x - offsetZ] +
1003	// coeff[1][x] * src[x - offsetY] +
1004	// coeff[2][x] * src[x - 1] +
1005	// coeff[3][x] * src[x] +
1006	// coeff[4][x] * src[x + 1] +
1007	// coeff[5][x] * src[x + offsetY] +
1008	// coeff[6][x] * src[x + offsetZ];
1009	}
1010
1011	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, endX);
1012	}
1013
1014	int flops()
1015	{
1016	return 40;
1017	}
1018	};
1019
1020	template<int DIM_X, int DIM_Y, int DIM_Z>
1021	class ExtendedVectorized3DFixed
1022	{
1023	public:
1024	static int coefficients()
1025	{
1026	return 13;
1027	}
1028
1029	inline void step(double coeff[13], double src, double *dst, int unusedOffsetY, int unusedOffsetZ, int startX, int endX)
1030	{
1031	const int SLICE_SIZE = DIM_X * DIM_Y;
1032	const int TOTAL_SIZE = DIM_X * DIM_Y * DIM_Z;
1033	const int offsetY = DIM_X;
1034	const int offsetZ = SLICE_SIZE;
1035
1036	int x = startX;
1037	ExtendedScalar3D scalarUpdater;
1038
1039	if ((x & 1) == 1) {
1040	scalarUpdater.step(coeff, src, dst, DIM_Y, offsetZ, x, x + 1);
1041	x += 1;
1042	}
1043
1044	__m128d same0 = _mm_load_pd(src + x + 0);
1045	__m128d neig0 = _mm_loadu_pd(src + x + 1);
1046
1047	int paddedEndX = endX - 7;
1048	for (; x < paddedEndX; x += 8) {
1049	__m128d same1 = _mm_load_pd(src + x + 2);
1050	__m128d same2 = _mm_load_pd(src + x + 4);
1051	__m128d same3 = _mm_load_pd(src + x + 6);
1052	__m128d same4 = _mm_load_pd(src + x + 8);
1053
1054	__m128d neig1 = _mm_shuffle_pd(same0, same1, (1 << 0) \| (0 << 2));
1055	__m128d neig2 = _mm_shuffle_pd(same1, same2, (1 << 0) \| (0 << 2));
1056	__m128d neig3 = _mm_shuffle_pd(same2, same3, (1 << 0) \| (0 << 2));
1057	__m128d neig4 = _mm_shuffle_pd(same3, same4, (1 << 0) \| (0 << 2));
1058
1059	same0 = _mm_mul_pd(same0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 3 + x + 0]));
1060	same1 = _mm_mul_pd(same1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 3 + x + 2]));
1061	same2 = _mm_mul_pd(same2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 3 + x + 4]));
1062	same3 = _mm_mul_pd(same3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 3 + x + 6]));
1063
1064	__m128d temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 2 + x + 0]));
1065	__m128d temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 2 + x + 2]));
1066	__m128d temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 2 + x + 4]));
1067	__m128d temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 2 + x + 6]));
1068
1069	same0 = _mm_add_pd(same0, temp1);
1070	same1 = _mm_add_pd(same1, temp2);
1071	same2 = _mm_add_pd(same2, temp3);
1072	same3 = _mm_add_pd(same3, temp4);
1073
1074	temp1 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 4 + x + 0]));
1075	temp2 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 4 + x + 2]));
1076	temp3 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 4 + x + 4]));
1077	temp4 = _mm_mul_pd(neig4, _mm_load_pd(&coeff[0][TOTAL_SIZE * 4 + x + 6]));
1078
1079	same0 = _mm_add_pd(same0, temp1);
1080	same1 = _mm_add_pd(same1, temp2);
1081	same2 = _mm_add_pd(same2, temp3);
1082	same3 = _mm_add_pd(same3, temp4);
1083
1084	neig0 = _mm_load_pd(src + x - offsetZ + 0);
1085	neig1 = _mm_load_pd(src + x - offsetZ + 2);
1086	neig2 = _mm_load_pd(src + x - offsetZ + 4);
1087	neig3 = _mm_load_pd(src + x - offsetZ + 6);
1088
1089	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 0 + x + 0]));
1090	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 0 + x + 2]));
1091	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 0 + x + 4]));
1092	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 0 + x + 6]));
1093
1094	same0 = _mm_add_pd(same0, temp1);
1095	same1 = _mm_add_pd(same1, temp2);
1096	same2 = _mm_add_pd(same2, temp3);
1097	same3 = _mm_add_pd(same3, temp4);
1098
1099	neig0 = _mm_load_pd(src + x - offsetY + 0);
1100	neig1 = _mm_load_pd(src + x - offsetY + 2);
1101	neig2 = _mm_load_pd(src + x - offsetY + 4);
1102	neig3 = _mm_load_pd(src + x - offsetY + 6);
1103
1104	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 1 + x + 0]));
1105	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 1 + x + 2]));
1106	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 1 + x + 4]));
1107	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 1 + x + 6]));
1108
1109	same0 = _mm_add_pd(same0, temp1);
1110	same1 = _mm_add_pd(same1, temp2);
1111	same2 = _mm_add_pd(same2, temp3);
1112	same3 = _mm_add_pd(same3, temp4);
1113
1114	neig0 = _mm_load_pd(src + x + offsetY + 0);
1115	neig1 = _mm_load_pd(src + x + offsetY + 2);
1116	neig2 = _mm_load_pd(src + x + offsetY + 4);
1117	neig3 = _mm_load_pd(src + x + offsetY + 6);
1118
1119	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 5 + x + 0]));
1120	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 5 + x + 2]));
1121	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 5 + x + 4]));
1122	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 5 + x + 6]));
1123
1124	same0 = _mm_add_pd(same0, temp1);
1125	same1 = _mm_add_pd(same1, temp2);
1126	same2 = _mm_add_pd(same2, temp3);
1127	same3 = _mm_add_pd(same3, temp4);
1128
1129	//xxxxxxxxxxxxx
1130	neig0 = _mm_load_pd(src + x + offsetZ + 0);
1131	neig1 = _mm_load_pd(src + x + offsetZ + 2);
1132	neig2 = _mm_load_pd(src + x + offsetZ + 4);
1133	neig3 = _mm_load_pd(src + x + offsetZ + 6);
1134
1135	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 6 + x + 0]));
1136	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 6 + x + 2]));
1137	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 6 + x + 4]));
1138	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 6 + x + 6]));
1139
1140	same0 = _mm_add_pd(same0, temp1);
1141	same1 = _mm_add_pd(same1, temp2);
1142	same2 = _mm_add_pd(same2, temp3);
1143	same3 = _mm_add_pd(same3, temp4);
1144
1145	//xxxxxxxxxxxxx
1146	neig0 = _mm_load_pd(src + x - offsetZ - offsetY + 0);
1147	neig1 = _mm_load_pd(src + x - offsetZ - offsetY + 2);
1148	neig2 = _mm_load_pd(src + x - offsetZ - offsetY + 4);
1149	neig3 = _mm_load_pd(src + x - offsetZ - offsetY + 6);
1150
1151	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 7 + x + 0]));
1152	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 7 + x + 2]));
1153	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 7 + x + 4]));
1154	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 7 + x + 6]));
1155
1156	same0 = _mm_add_pd(same0, temp1);
1157	same1 = _mm_add_pd(same1, temp2);
1158	same2 = _mm_add_pd(same2, temp3);
1159	same3 = _mm_add_pd(same3, temp4);
1160
1161	//xxxxxxxxxxxxx
1162	neig0 = _mm_load_pd(src + x - offsetZ + offsetY + 0);
1163	neig1 = _mm_load_pd(src + x - offsetZ + offsetY + 2);
1164	neig2 = _mm_load_pd(src + x - offsetZ + offsetY + 4);
1165	neig3 = _mm_load_pd(src + x - offsetZ + offsetY + 6);
1166
1167	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 8 + x + 0]));
1168	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 8 + x + 2]));
1169	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 8 + x + 4]));
1170	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 8 + x + 6]));
1171
1172	same0 = _mm_add_pd(same0, temp1);
1173	same1 = _mm_add_pd(same1, temp2);
1174	same2 = _mm_add_pd(same2, temp3);
1175	same3 = _mm_add_pd(same3, temp4);
1176
1177	//xxxxxxxxxxxxx
1178	neig0 = _mm_load_pd(src + x + offsetZ - offsetY + 0);
1179	neig1 = _mm_load_pd(src + x + offsetZ - offsetY + 2);
1180	neig2 = _mm_load_pd(src + x + offsetZ - offsetY + 4);
1181	neig3 = _mm_load_pd(src + x + offsetZ - offsetY + 6);
1182
1183	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 9 + x + 0]));
1184	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 9 + x + 2]));
1185	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 9 + x + 4]));
1186	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 9 + x + 6]));
1187
1188	same0 = _mm_add_pd(same0, temp1);
1189	same1 = _mm_add_pd(same1, temp2);
1190	same2 = _mm_add_pd(same2, temp3);
1191	same3 = _mm_add_pd(same3, temp4);
1192
1193	//xxxxxxxxxxxxx
1194	neig0 = _mm_load_pd(src + x + offsetZ + offsetY + 0);
1195	neig1 = _mm_load_pd(src + x + offsetZ + offsetY + 2);
1196	neig2 = _mm_load_pd(src + x + offsetZ + offsetY + 4);
1197	neig3 = _mm_load_pd(src + x + offsetZ + offsetY + 6);
1198
1199	temp1 = _mm_mul_pd(neig0, _mm_load_pd(&coeff[0][TOTAL_SIZE * 10 + x + 0]));
1200	temp2 = _mm_mul_pd(neig1, _mm_load_pd(&coeff[0][TOTAL_SIZE * 10 + x + 2]));
1201	temp3 = _mm_mul_pd(neig2, _mm_load_pd(&coeff[0][TOTAL_SIZE * 10 + x + 4]));
1202	temp4 = _mm_mul_pd(neig3, _mm_load_pd(&coeff[0][TOTAL_SIZE * 10 + x + 6]));
1203
1204	same0 = _mm_add_pd(same0, temp1);
1205	same1 = _mm_add_pd(same1, temp2);
1206	same2 = _mm_add_pd(same2, temp3);
1207	same3 = _mm_add_pd(same3, temp4);
1208
1209	//xxxxxxxxxxxxx
1210	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ - offsetY + 0);
1211	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ - offsetY + 2);
1212	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ - offsetY + 4);
1213	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ - offsetY + 6);
1214
1215	same0 = _mm_add_pd(same0, neig0);
1216	same1 = _mm_add_pd(same1, neig1);
1217	same2 = _mm_add_pd(same2, neig2);
1218	same3 = _mm_add_pd(same3, neig3);
1219
1220	//xxxxxxxxxxxxx
1221	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + 0);
1222	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + 2);
1223	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + 4);
1224	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + 6);
1225
1226	same0 = _mm_add_pd(same0, neig0);
1227	same1 = _mm_add_pd(same1, neig1);
1228	same2 = _mm_add_pd(same2, neig2);
1229	same3 = _mm_add_pd(same3, neig3);
1230
1231	//xxxxxxxxxxxxx
1232	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + offsetY + 0);
1233	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + offsetY + 2);
1234	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + offsetY + 4);
1235	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetZ + offsetY + 6);
1236
1237	same0 = _mm_add_pd(same0, neig0);
1238	same1 = _mm_add_pd(same1, neig1);
1239	same2 = _mm_add_pd(same2, neig2);
1240	same3 = _mm_add_pd(same3, neig3);
1241
1242	//xxxxxxxxxxxxx
1243	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetY + 0);
1244	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetY + 2);
1245	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetY + 4);
1246	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x - offsetY + 6);
1247
1248	same0 = _mm_add_pd(same0, neig0);
1249	same1 = _mm_add_pd(same1, neig1);
1250	same2 = _mm_add_pd(same2, neig2);
1251	same3 = _mm_add_pd(same3, neig3);
1252
1253	//xxxxxxxxxxxxx
1254	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + offsetY + x + 0);
1255	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + offsetY + x + 2);
1256	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + offsetY + x + 4);
1257	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + offsetY + x + 6);
1258
1259	same0 = _mm_add_pd(same0, neig0);
1260	same1 = _mm_add_pd(same1, neig1);
1261	same2 = _mm_add_pd(same2, neig2);
1262	same3 = _mm_add_pd(same3, neig3);
1263
1264	//xxxxxxxxxxxxx
1265	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + 0);
1266	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + 2);
1267	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + 4);
1268	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + 6);
1269
1270	same0 = _mm_add_pd(same0, neig0);
1271	same1 = _mm_add_pd(same1, neig1);
1272	same2 = _mm_add_pd(same2, neig2);
1273	same3 = _mm_add_pd(same3, neig3);
1274
1275	//xxxxxxxxxxxxx
1276	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ - offsetY + 0);
1277	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ - offsetY + 2);
1278	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ - offsetY + 4);
1279	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ - offsetY + 6);
1280
1281	same0 = _mm_add_pd(same0, neig0);
1282	same1 = _mm_add_pd(same1, neig1);
1283	same2 = _mm_add_pd(same2, neig2);
1284	same3 = _mm_add_pd(same3, neig3);
1285
1286	//xxxxxxxxxxxxx
1287	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + 0);
1288	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + 2);
1289	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + 4);
1290	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + 6);
1291
1292	same0 = _mm_add_pd(same0, neig0);
1293	same1 = _mm_add_pd(same1, neig1);
1294	same2 = _mm_add_pd(same2, neig2);
1295	same3 = _mm_add_pd(same3, neig3);
1296
1297	//xxxxxxxxxxxxx
1298	neig0 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + offsetY + 0);
1299	neig1 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + offsetY + 2);
1300	neig2 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + offsetY + 4);
1301	neig3 = _mm_load_pd(coeff[0] + 11 * TOTAL_SIZE + x + offsetZ + offsetY + 6);
1302
1303	same0 = _mm_add_pd(same0, neig0);
1304	same1 = _mm_add_pd(same1, neig1);
1305	same2 = _mm_add_pd(same2, neig2);
1306	same3 = _mm_add_pd(same3, neig3);
1307
1308	//xxxxxxxxxxxxx
1309	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ - offsetY + 0);
1310	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ - offsetY + 2);
1311	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ - offsetY + 4);
1312	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ - offsetY + 6);
1313
1314	same0 = _mm_add_pd(same0, neig0);
1315	same1 = _mm_add_pd(same1, neig1);
1316	same2 = _mm_add_pd(same2, neig2);
1317	same3 = _mm_add_pd(same3, neig3);
1318
1319	//xxxxxxxxxxxxx
1320	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + 0);
1321	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + 2);
1322	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + 4);
1323	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + 6);
1324
1325	same0 = _mm_add_pd(same0, neig0);
1326	same1 = _mm_add_pd(same1, neig1);
1327	same2 = _mm_add_pd(same2, neig2);
1328	same3 = _mm_add_pd(same3, neig3);
1329
1330	//xxxxxxxxxxxxx
1331	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + offsetY + 0);
1332	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + offsetY + 2);
1333	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + offsetY + 4);
1334	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetZ + offsetY + 6);
1335
1336	same0 = _mm_add_pd(same0, neig0);
1337	same1 = _mm_add_pd(same1, neig1);
1338	same2 = _mm_add_pd(same2, neig2);
1339	same3 = _mm_add_pd(same3, neig3);
1340
1341	//xxxxxxxxxxxxx
1342	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetY + 0);
1343	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetY + 2);
1344	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetY + 4);
1345	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x - offsetY + 6);
1346
1347	same0 = _mm_add_pd(same0, neig0);
1348	same1 = _mm_add_pd(same1, neig1);
1349	same2 = _mm_add_pd(same2, neig2);
1350	same3 = _mm_add_pd(same3, neig3);
1351
1352	//xxxxxxxxxxxxx
1353	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + offsetY + x + 0);
1354	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + offsetY + x + 2);
1355	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + offsetY + x + 4);
1356	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + offsetY + x + 6);
1357
1358	same0 = _mm_add_pd(same0, neig0);
1359	same1 = _mm_add_pd(same1, neig1);
1360	same2 = _mm_add_pd(same2, neig2);
1361	same3 = _mm_add_pd(same3, neig3);
1362
1363	//xxxxxxxxxxxxx
1364	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + 0);
1365	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + 2);
1366	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + 4);
1367	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + 6);
1368
1369	same0 = _mm_add_pd(same0, neig0);
1370	same1 = _mm_add_pd(same1, neig1);
1371	same2 = _mm_add_pd(same2, neig2);
1372	same3 = _mm_add_pd(same3, neig3);
1373
1374	//xxxxxxxxxxxxx
1375	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ - offsetY + 0);
1376	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ - offsetY + 2);
1377	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ - offsetY + 4);
1378	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ - offsetY + 6);
1379
1380	same0 = _mm_add_pd(same0, neig0);
1381	same1 = _mm_add_pd(same1, neig1);
1382	same2 = _mm_add_pd(same2, neig2);
1383	same3 = _mm_add_pd(same3, neig3);
1384
1385	//xxxxxxxxxxxxx
1386	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + 0);
1387	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + 2);
1388	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + 4);
1389	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + 6);
1390
1391	same0 = _mm_add_pd(same0, neig0);
1392	same1 = _mm_add_pd(same1, neig1);
1393	same2 = _mm_add_pd(same2, neig2);
1394	same3 = _mm_add_pd(same3, neig3);
1395
1396	//xxxxxxxxxxxxx
1397	neig0 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + offsetY + 0);
1398	neig1 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + offsetY + 2);
1399	neig2 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + offsetY + 4);
1400	neig3 = _mm_load_pd(coeff[0] + 12 * TOTAL_SIZE + x + offsetZ + offsetY + 6);
1401
1402	same0 = _mm_add_pd(same0, neig0);
1403	same1 = _mm_add_pd(same1, neig1);
1404	same2 = _mm_add_pd(same2, neig2);
1405	same3 = _mm_add_pd(same3, neig3);
1406
1407	//yyyyyyyyyyyyy
1408	_mm_store_pd(dst + x + 0, same0);
1409	_mm_store_pd(dst + x + 2, same1);
1410	_mm_store_pd(dst + x + 4, same2);
1411	_mm_store_pd(dst + x + 6, same3);
1412
1413	same0 = same4;
1414	neig0 = neig4;
1415
1416	// dst[x] =
1417	// coeff[0][x] * src[x - offsetZ] +
1418	// coeff[1][x] * src[x - offsetY] +
1419	// coeff[2][x] * src[x - 1] +
1420	// coeff[3][x] * src[x] +
1421	// coeff[4][x] * src[x + 1] +
1422	// coeff[5][x] * src[x + offsetY] +
1423	// coeff[6][x] * src[x + offsetZ];
1424	}
1425
1426	scalarUpdater.step(coeff, src, dst, offsetY, offsetZ, x, endX);
1427	}
1428
1429	int flops()
1430	{
1431	return 40;
1432	}
1433	};
1434
1435
1436	int main(int argc, char *argv[])
1437	{
1438	// Benchmark<Scalar3D, 3>().exercise();
1439	// Benchmark<Vectorized3D, 3>().exercise();
1440	// Benchmark<VectorizedSSEMelbourneShuffle2D>().exercise();
1441	Benchmark<ExtendedVectorized3D, 3>().exercise();
1442	// Benchmark<ExtendedVectorized3DFixed, 3>().exercise();
1443	// Benchmark<Jacobi3D, 3>().exercise();
1444
1445
1446	// std::vector<cl::Platform> platforms;
1447	// cl::Platform::get(&platforms);
1448
1449	// for (int i = 0; i < platforms.size(); ++i) {
1450	// std::string str;
1451	// platforms[i].getInfo(CL_PLATFORM_NAME, &str);
1452	// std::cout << "Platform[" << i << "] = " << str << std::endl;
1453	// }
1454
1455	return 0;
1456	}

Note: See TracBrowser for help on using the browser.

Context Navigation

root/src/testbed/testbed.cpp @ 177:ee6efdda9ac6

Download in other formats: