diff options
| author | upstream source tree <ports@midipix.org> | 2015-03-15 20:14:05 -0400 |
|---|---|---|
| committer | upstream source tree <ports@midipix.org> | 2015-03-15 20:14:05 -0400 |
| commit | 554fd8c5195424bdbcabf5de30fdc183aba391bd (patch) | |
| tree | 976dc5ab7fddf506dadce60ae936f43f58787092 /gcc/config/arm/neon.md | |
| download | cbb-gcc-4.6.4-upstream.tar.bz2 cbb-gcc-4.6.4-upstream.tar.xz | |
obtained gcc-4.6.4.tar.bz2 from upstream website;upstream
verified gcc-4.6.4.tar.bz2.sig;
imported gcc-4.6.4 source tree from verified upstream tarball.
downloading a git-generated archive based on the 'upstream' tag
should provide you with a source tree that is binary identical
to the one extracted from the above tarball.
if you have obtained the source via the command 'git clone',
however, do note that line-endings of files in your working
directory might differ from line-endings of the respective
files in the upstream repository.
Diffstat (limited to 'gcc/config/arm/neon.md')
| -rw-r--r-- | gcc/config/arm/neon.md | 5476 |
1 files changed, 5476 insertions, 0 deletions
diff --git a/gcc/config/arm/neon.md b/gcc/config/arm/neon.md new file mode 100644 index 000000000..247dc1ff4 --- /dev/null +++ b/gcc/config/arm/neon.md @@ -0,0 +1,5476 @@ +;; ARM NEON coprocessor Machine Description +;; Copyright (C) 2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc. +;; Written by CodeSourcery. +;; +;; This file is part of GCC. +;; +;; GCC is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 3, or (at your option) +;; any later version. +;; +;; GCC is distributed in the hope that it will be useful, but +;; WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;; General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with GCC; see the file COPYING3. If not see +;; <http://www.gnu.org/licenses/>. + +;; Constants for unspecs. +(define_constants + [(UNSPEC_ASHIFT_SIGNED 65) + (UNSPEC_ASHIFT_UNSIGNED 66) + (UNSPEC_VABD 69) + (UNSPEC_VABDL 70) + (UNSPEC_VADD 72) + (UNSPEC_VADDHN 73) + (UNSPEC_VADDL 74) + (UNSPEC_VADDW 75) + (UNSPEC_VBSL 78) + (UNSPEC_VCAGE 79) + (UNSPEC_VCAGT 80) + (UNSPEC_VCEQ 81) + (UNSPEC_VCGE 82) + (UNSPEC_VCGT 83) + (UNSPEC_VCLS 84) + (UNSPEC_VCVT 88) + (UNSPEC_VCVT_N 89) + (UNSPEC_VEXT 93) + (UNSPEC_VHADD 97) + (UNSPEC_VHSUB 98) + (UNSPEC_VLD1 99) + (UNSPEC_VLD1_DUP 100) + (UNSPEC_VLD1_LANE 101) + (UNSPEC_VLD2 102) + (UNSPEC_VLD2_DUP 103) + (UNSPEC_VLD2_LANE 104) + (UNSPEC_VLD3 105) + (UNSPEC_VLD3A 106) + (UNSPEC_VLD3B 107) + (UNSPEC_VLD3_DUP 108) + (UNSPEC_VLD3_LANE 109) + (UNSPEC_VLD4 110) + (UNSPEC_VLD4A 111) + (UNSPEC_VLD4B 112) + (UNSPEC_VLD4_DUP 113) + (UNSPEC_VLD4_LANE 114) + (UNSPEC_VMAX 115) + (UNSPEC_VMIN 116) + (UNSPEC_VMLA 117) + (UNSPEC_VMLAL 118) + (UNSPEC_VMLA_LANE 119) + (UNSPEC_VMLAL_LANE 120) + (UNSPEC_VMLS 121) + (UNSPEC_VMLSL 122) + (UNSPEC_VMLS_LANE 123) + (UNSPEC_VMLSL_LANE 124) + (UNSPEC_VMOVL 125) + (UNSPEC_VMOVN 126) + (UNSPEC_VMUL 127) + (UNSPEC_VMULL 128) + (UNSPEC_VMUL_LANE 129) + (UNSPEC_VMULL_LANE 130) + (UNSPEC_VPADAL 135) + (UNSPEC_VPADD 136) + (UNSPEC_VPADDL 137) + (UNSPEC_VPMAX 138) + (UNSPEC_VPMIN 139) + (UNSPEC_VPSMAX 140) + (UNSPEC_VPSMIN 141) + (UNSPEC_VPUMAX 142) + (UNSPEC_VPUMIN 143) + (UNSPEC_VQABS 144) + (UNSPEC_VQADD 145) + (UNSPEC_VQDMLAL 146) + (UNSPEC_VQDMLAL_LANE 147) + (UNSPEC_VQDMLSL 148) + (UNSPEC_VQDMLSL_LANE 149) + (UNSPEC_VQDMULH 150) + (UNSPEC_VQDMULH_LANE 151) + (UNSPEC_VQDMULL 152) + (UNSPEC_VQDMULL_LANE 153) + (UNSPEC_VQMOVN 154) + (UNSPEC_VQMOVUN 155) + (UNSPEC_VQNEG 156) + (UNSPEC_VQSHL 157) + (UNSPEC_VQSHL_N 158) + (UNSPEC_VQSHLU_N 159) + (UNSPEC_VQSHRN_N 160) + (UNSPEC_VQSHRUN_N 161) + (UNSPEC_VQSUB 162) + (UNSPEC_VRECPE 163) + (UNSPEC_VRECPS 164) + (UNSPEC_VREV16 165) + (UNSPEC_VREV32 166) + (UNSPEC_VREV64 167) + (UNSPEC_VRSQRTE 168) + (UNSPEC_VRSQRTS 169) + (UNSPEC_VSHL 171) + (UNSPEC_VSHLL_N 172) + (UNSPEC_VSHL_N 173) + (UNSPEC_VSHR_N 174) + (UNSPEC_VSHRN_N 175) + (UNSPEC_VSLI 176) + (UNSPEC_VSRA_N 177) + (UNSPEC_VSRI 178) + (UNSPEC_VST1 179) + (UNSPEC_VST1_LANE 180) + (UNSPEC_VST2 181) + (UNSPEC_VST2_LANE 182) + (UNSPEC_VST3 183) + (UNSPEC_VST3A 184) + (UNSPEC_VST3B 185) + (UNSPEC_VST3_LANE 186) + (UNSPEC_VST4 187) + (UNSPEC_VST4A 188) + (UNSPEC_VST4B 189) + (UNSPEC_VST4_LANE 190) + (UNSPEC_VSTRUCTDUMMY 191) + (UNSPEC_VSUB 192) + (UNSPEC_VSUBHN 193) + (UNSPEC_VSUBL 194) + (UNSPEC_VSUBW 195) + (UNSPEC_VTBL 196) + (UNSPEC_VTBX 197) + (UNSPEC_VTRN1 198) + (UNSPEC_VTRN2 199) + (UNSPEC_VTST 200) + (UNSPEC_VUZP1 201) + (UNSPEC_VUZP2 202) + (UNSPEC_VZIP1 203) + (UNSPEC_VZIP2 204) + (UNSPEC_MISALIGNED_ACCESS 205) + (UNSPEC_VCLE 206) + (UNSPEC_VCLT 207)]) + + +;; Attribute used to permit string comparisons against <VQH_mnem> in +;; neon_type attribute definitions. +(define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd")) + +(define_insn "*neon_mov<mode>" + [(set (match_operand:VD 0 "nonimmediate_operand" + "=w,Uv,w, w, ?r,?w,?r,?r, ?Us") + (match_operand:VD 1 "general_operand" + " w,w, Dn,Uvi, w, r, r, Usi,r"))] + "TARGET_NEON + && (register_operand (operands[0], <MODE>mode) + || register_operand (operands[1], <MODE>mode))" +{ + if (which_alternative == 2) + { + int width, is_valid; + static char templ[40]; + + is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode, + &operands[1], &width); + + gcc_assert (is_valid != 0); + + if (width == 0) + return "vmov.f32\t%P0, %1 @ <mode>"; + else + sprintf (templ, "vmov.i%d\t%%P0, %%1 @ <mode>", width); + + return templ; + } + + /* FIXME: If the memory layout is changed in big-endian mode, output_move_vfp + below must be changed to output_move_neon (which will use the + element/structure loads/stores), and the constraint changed to 'Um' instead + of 'Uv'. */ + + switch (which_alternative) + { + case 0: return "vmov\t%P0, %P1 @ <mode>"; + case 1: case 3: return output_move_vfp (operands); + case 2: gcc_unreachable (); + case 4: return "vmov\t%Q0, %R0, %P1 @ <mode>"; + case 5: return "vmov\t%P0, %Q1, %R1 @ <mode>"; + default: return output_move_double (operands); + } +} + [(set_attr "neon_type" "neon_int_1,*,neon_vmov,*,neon_mrrc,neon_mcr_2_mcrr,*,*,*") + (set_attr "type" "*,f_stored,*,f_loadd,*,*,alu,load2,store2") + (set_attr "insn" "*,*,*,*,*,*,mov,*,*") + (set_attr "length" "4,4,4,4,4,4,8,8,8") + (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*") + (set_attr "neg_pool_range" "*,*,*,1008,*,*,*,1008,*")]) + +(define_insn "*neon_mov<mode>" + [(set (match_operand:VQXMOV 0 "nonimmediate_operand" + "=w,Un,w, w, ?r,?w,?r,?r, ?Us") + (match_operand:VQXMOV 1 "general_operand" + " w,w, Dn,Uni, w, r, r, Usi, r"))] + "TARGET_NEON + && (register_operand (operands[0], <MODE>mode) + || register_operand (operands[1], <MODE>mode))" +{ + if (which_alternative == 2) + { + int width, is_valid; + static char templ[40]; + + is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode, + &operands[1], &width); + + gcc_assert (is_valid != 0); + + if (width == 0) + return "vmov.f32\t%q0, %1 @ <mode>"; + else + sprintf (templ, "vmov.i%d\t%%q0, %%1 @ <mode>", width); + + return templ; + } + + switch (which_alternative) + { + case 0: return "vmov\t%q0, %q1 @ <mode>"; + case 1: case 3: return output_move_neon (operands); + case 2: gcc_unreachable (); + case 4: return "vmov\t%Q0, %R0, %e1 @ <mode>\;vmov\t%J0, %K0, %f1"; + case 5: return "vmov\t%e0, %Q1, %R1 @ <mode>\;vmov\t%f0, %J1, %K1"; + default: return output_move_quad (operands); + } +} + [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_vmov,neon_ldm_2,\ + neon_mrrc,neon_mcr_2_mcrr,*,*,*") + (set_attr "type" "*,*,*,*,*,*,alu,load4,store4") + (set_attr "insn" "*,*,*,*,*,*,mov,*,*") + (set_attr "length" "4,8,4,8,8,8,16,8,16") + (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*") + (set_attr "neg_pool_range" "*,*,*,1008,*,*,*,1008,*")]) + +(define_expand "movti" + [(set (match_operand:TI 0 "nonimmediate_operand" "") + (match_operand:TI 1 "general_operand" ""))] + "TARGET_NEON" +{ + if (can_create_pseudo_p ()) + { + if (GET_CODE (operands[0]) != REG) + operands[1] = force_reg (TImode, operands[1]); + } +}) + +(define_expand "mov<mode>" + [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "") + (match_operand:VSTRUCT 1 "general_operand" ""))] + "TARGET_NEON" +{ + if (can_create_pseudo_p ()) + { + if (GET_CODE (operands[0]) != REG) + operands[1] = force_reg (<MODE>mode, operands[1]); + } +}) + +(define_insn "*neon_mov<mode>" + [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w") + (match_operand:VSTRUCT 1 "general_operand" " w,w, Ut"))] + "TARGET_NEON + && (register_operand (operands[0], <MODE>mode) + || register_operand (operands[1], <MODE>mode))" +{ + switch (which_alternative) + { + case 0: return "#"; + case 1: case 2: return output_move_neon (operands); + default: gcc_unreachable (); + } +} + [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_ldm_2") + (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))]) + +(define_split + [(set (match_operand:EI 0 "s_register_operand" "") + (match_operand:EI 1 "s_register_operand" ""))] + "TARGET_NEON && reload_completed" + [(set (match_dup 0) (match_dup 1)) + (set (match_dup 2) (match_dup 3))] +{ + int rdest = REGNO (operands[0]); + int rsrc = REGNO (operands[1]); + rtx dest[2], src[2]; + + dest[0] = gen_rtx_REG (TImode, rdest); + src[0] = gen_rtx_REG (TImode, rsrc); + dest[1] = gen_rtx_REG (DImode, rdest + 4); + src[1] = gen_rtx_REG (DImode, rsrc + 4); + + neon_disambiguate_copy (operands, dest, src, 2); +}) + +(define_split + [(set (match_operand:OI 0 "s_register_operand" "") + (match_operand:OI 1 "s_register_operand" ""))] + "TARGET_NEON && reload_completed" + [(set (match_dup 0) (match_dup 1)) + (set (match_dup 2) (match_dup 3))] +{ + int rdest = REGNO (operands[0]); + int rsrc = REGNO (operands[1]); + rtx dest[2], src[2]; + + dest[0] = gen_rtx_REG (TImode, rdest); + src[0] = gen_rtx_REG (TImode, rsrc); + dest[1] = gen_rtx_REG (TImode, rdest + 4); + src[1] = gen_rtx_REG (TImode, rsrc + 4); + + neon_disambiguate_copy (operands, dest, src, 2); +}) + +(define_split + [(set (match_operand:CI 0 "s_register_operand" "") + (match_operand:CI 1 "s_register_operand" ""))] + "TARGET_NEON && reload_completed" + [(set (match_dup 0) (match_dup 1)) + (set (match_dup 2) (match_dup 3)) + (set (match_dup 4) (match_dup 5))] +{ + int rdest = REGNO (operands[0]); + int rsrc = REGNO (operands[1]); + rtx dest[3], src[3]; + + dest[0] = gen_rtx_REG (TImode, rdest); + src[0] = gen_rtx_REG (TImode, rsrc); + dest[1] = gen_rtx_REG (TImode, rdest + 4); + src[1] = gen_rtx_REG (TImode, rsrc + 4); + dest[2] = gen_rtx_REG (TImode, rdest + 8); + src[2] = gen_rtx_REG (TImode, rsrc + 8); + + neon_disambiguate_copy (operands, dest, src, 3); +}) + +(define_split + [(set (match_operand:XI 0 "s_register_operand" "") + (match_operand:XI 1 "s_register_operand" ""))] + "TARGET_NEON && reload_completed" + [(set (match_dup 0) (match_dup 1)) + (set (match_dup 2) (match_dup 3)) + (set (match_dup 4) (match_dup 5)) + (set (match_dup 6) (match_dup 7))] +{ + int rdest = REGNO (operands[0]); + int rsrc = REGNO (operands[1]); + rtx dest[4], src[4]; + + dest[0] = gen_rtx_REG (TImode, rdest); + src[0] = gen_rtx_REG (TImode, rsrc); + dest[1] = gen_rtx_REG (TImode, rdest + 4); + src[1] = gen_rtx_REG (TImode, rsrc + 4); + dest[2] = gen_rtx_REG (TImode, rdest + 8); + src[2] = gen_rtx_REG (TImode, rsrc + 8); + dest[3] = gen_rtx_REG (TImode, rdest + 12); + src[3] = gen_rtx_REG (TImode, rsrc + 12); + + neon_disambiguate_copy (operands, dest, src, 4); +}) + +(define_expand "movmisalign<mode>" + [(set (match_operand:VDQX 0 "nonimmediate_operand" "") + (unspec:VDQX [(match_operand:VDQX 1 "general_operand" "")] + UNSPEC_MISALIGNED_ACCESS))] + "TARGET_NEON && !BYTES_BIG_ENDIAN" +{ + /* This pattern is not permitted to fail during expansion: if both arguments + are non-registers (e.g. memory := constant, which can be created by the + auto-vectorizer), force operand 1 into a register. */ + if (!s_register_operand (operands[0], <MODE>mode) + && !s_register_operand (operands[1], <MODE>mode)) + operands[1] = force_reg (<MODE>mode, operands[1]); +}) + +(define_insn "*movmisalign<mode>_neon_store" + [(set (match_operand:VDX 0 "memory_operand" "=Um") + (unspec:VDX [(match_operand:VDX 1 "s_register_operand" " w")] + UNSPEC_MISALIGNED_ACCESS))] + "TARGET_NEON && !BYTES_BIG_ENDIAN" + "vst1.<V_sz_elem>\t{%P1}, %A0" + [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]) + +(define_insn "*movmisalign<mode>_neon_load" + [(set (match_operand:VDX 0 "s_register_operand" "=w") + (unspec:VDX [(match_operand:VDX 1 "memory_operand" " Um")] + UNSPEC_MISALIGNED_ACCESS))] + "TARGET_NEON && !BYTES_BIG_ENDIAN" + "vld1.<V_sz_elem>\t{%P0}, %A1" + [(set_attr "neon_type" "neon_vld1_1_2_regs")]) + +(define_insn "*movmisalign<mode>_neon_store" + [(set (match_operand:VQX 0 "memory_operand" "=Um") + (unspec:VQX [(match_operand:VQX 1 "s_register_operand" " w")] + UNSPEC_MISALIGNED_ACCESS))] + "TARGET_NEON && !BYTES_BIG_ENDIAN" + "vst1.<V_sz_elem>\t{%q1}, %A0" + [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]) + +(define_insn "*movmisalign<mode>_neon_load" + [(set (match_operand:VQX 0 "s_register_operand" "=w") + (unspec:VQX [(match_operand:VQX 1 "memory_operand" " Um")] + UNSPEC_MISALIGNED_ACCESS))] + "TARGET_NEON && !BYTES_BIG_ENDIAN" + "vld1.<V_sz_elem>\t{%q0}, %A1" + [(set_attr "neon_type" "neon_vld1_1_2_regs")]) + +(define_insn "vec_set<mode>_internal" + [(set (match_operand:VD 0 "s_register_operand" "=w") + (vec_merge:VD + (vec_duplicate:VD + (match_operand:<V_elem> 1 "s_register_operand" "r")) + (match_operand:VD 3 "s_register_operand" "0") + (match_operand:SI 2 "immediate_operand" "i")))] + "TARGET_NEON" +{ + int elt = ffs ((int) INTVAL (operands[2])) - 1; + if (BYTES_BIG_ENDIAN) + elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt; + operands[2] = GEN_INT (elt); + + return "vmov%?.<V_sz_elem>\t%P0[%c2], %1"; +} + [(set_attr "predicable" "yes") + (set_attr "neon_type" "neon_mcr")]) + +(define_insn "vec_set<mode>_internal" + [(set (match_operand:VQ 0 "s_register_operand" "=w") + (vec_merge:VQ + (vec_duplicate:VQ + (match_operand:<V_elem> 1 "s_register_operand" "r")) + (match_operand:VQ 3 "s_register_operand" "0") + (match_operand:SI 2 "immediate_operand" "i")))] + "TARGET_NEON" +{ + HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1; + int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2; + int elt = elem % half_elts; + int hi = (elem / half_elts) * 2; + int regno = REGNO (operands[0]); + + if (BYTES_BIG_ENDIAN) + elt = half_elts - 1 - elt; + + operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi); + operands[2] = GEN_INT (elt); + + return "vmov%?.<V_sz_elem>\t%P0[%c2], %1"; +} + [(set_attr "predicable" "yes") + (set_attr "neon_type" "neon_mcr")] +) + +(define_insn "vec_setv2di_internal" + [(set (match_operand:V2DI 0 "s_register_operand" "=w") + (vec_merge:V2DI + (vec_duplicate:V2DI + (match_operand:DI 1 "s_register_operand" "r")) + (match_operand:V2DI 3 "s_register_operand" "0") + (match_operand:SI 2 "immediate_operand" "i")))] + "TARGET_NEON" +{ + HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1; + int regno = REGNO (operands[0]) + 2 * elem; + + operands[0] = gen_rtx_REG (DImode, regno); + + return "vmov%?\t%P0, %Q1, %R1"; +} + [(set_attr "predicable" "yes") + (set_attr "neon_type" "neon_mcr_2_mcrr")] +) + +(define_expand "vec_set<mode>" + [(match_operand:VDQ 0 "s_register_operand" "") + (match_operand:<V_elem> 1 "s_register_operand" "") + (match_operand:SI 2 "immediate_operand" "")] + "TARGET_NEON" +{ + HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]); + emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1], + GEN_INT (elem), operands[0])); + DONE; +}) + +(define_insn "vec_extract<mode>" + [(set (match_operand:<V_elem> 0 "s_register_operand" "=r") + (vec_select:<V_elem> + (match_operand:VD 1 "s_register_operand" "w") + (parallel [(match_operand:SI 2 "immediate_operand" "i")])))] + "TARGET_NEON" +{ + if (BYTES_BIG_ENDIAN) + { + int elt = INTVAL (operands[2]); + elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt; + operands[2] = GEN_INT (elt); + } + return "vmov%?.<V_uf_sclr>\t%0, %P1[%c2]"; +} + [(set_attr "predicable" "yes") + (set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "vec_extract<mode>" + [(set (match_operand:<V_elem> 0 "s_register_operand" "=r") + (vec_select:<V_elem> + (match_operand:VQ 1 "s_register_operand" "w") + (parallel [(match_operand:SI 2 "immediate_operand" "i")])))] + "TARGET_NEON" +{ + int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2; + int elt = INTVAL (operands[2]) % half_elts; + int hi = (INTVAL (operands[2]) / half_elts) * 2; + int regno = REGNO (operands[1]); + + if (BYTES_BIG_ENDIAN) + elt = half_elts - 1 - elt; + + operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi); + operands[2] = GEN_INT (elt); + + return "vmov%?.<V_uf_sclr>\t%0, %P1[%c2]"; +} + [(set_attr "predicable" "yes") + (set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "vec_extractv2di" + [(set (match_operand:DI 0 "s_register_operand" "=r") + (vec_select:DI + (match_operand:V2DI 1 "s_register_operand" "w") + (parallel [(match_operand:SI 2 "immediate_operand" "i")])))] + "TARGET_NEON" +{ + int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]); + + operands[1] = gen_rtx_REG (DImode, regno); + + return "vmov%?\t%Q0, %R0, %P1 @ v2di"; +} + [(set_attr "predicable" "yes") + (set_attr "neon_type" "neon_int_1")] +) + +(define_expand "vec_init<mode>" + [(match_operand:VDQ 0 "s_register_operand" "") + (match_operand 1 "" "")] + "TARGET_NEON" +{ + neon_expand_vector_init (operands[0], operands[1]); + DONE; +}) + +;; Doubleword and quadword arithmetic. + +;; NOTE: some other instructions also support 64-bit integer +;; element size, which we could potentially use for "long long" operations. + +(define_insn "*add<mode>3_neon" + [(set (match_operand:VDQ 0 "s_register_operand" "=w") + (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w") + (match_operand:VDQ 2 "s_register_operand" "w")))] + "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" + "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_1")))] +) + +(define_insn "adddi3_neon" + [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r") + (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0") + (match_operand:DI 2 "s_register_operand" "w,r,0"))) + (clobber (reg:CC CC_REGNUM))] + "TARGET_NEON" +{ + switch (which_alternative) + { + case 0: return "vadd.i64\t%P0, %P1, %P2"; + case 1: return "#"; + case 2: return "#"; + default: gcc_unreachable (); + } +} + [(set_attr "neon_type" "neon_int_1,*,*") + (set_attr "conds" "*,clob,clob") + (set_attr "length" "*,8,8")] +) + +(define_insn "*sub<mode>3_neon" + [(set (match_operand:VDQ 0 "s_register_operand" "=w") + (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w") + (match_operand:VDQ 2 "s_register_operand" "w")))] + "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" + "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_2")))] +) + +(define_insn "subdi3_neon" + [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r") + (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0") + (match_operand:DI 2 "s_register_operand" "w,r,0,0"))) + (clobber (reg:CC CC_REGNUM))] + "TARGET_NEON" +{ + switch (which_alternative) + { + case 0: return "vsub.i64\t%P0, %P1, %P2"; + case 1: /* fall through */ + case 2: /* fall through */ + case 3: return "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2"; + default: gcc_unreachable (); + } +} + [(set_attr "neon_type" "neon_int_2,*,*,*") + (set_attr "conds" "*,clob,clob,clob") + (set_attr "length" "*,8,8,8")] +) + +(define_insn "*mul<mode>3_neon" + [(set (match_operand:VDQ 0 "s_register_operand" "=w") + (mult:VDQ (match_operand:VDQ 1 "s_register_operand" "w") + (match_operand:VDQ 2 "s_register_operand" "w")))] + "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" + "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (if_then_else + (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long") + (const_string "neon_mul_qqq_8_16_32_ddd_32")) + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_qqq_8_16_32_ddd_32") + (const_string "neon_mul_qqq_8_16_32_ddd_32")))))] +) + +(define_insn "mul<mode>3add<mode>_neon" + [(set (match_operand:VDQ 0 "s_register_operand" "=w") + (plus:VDQ (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w") + (match_operand:VDQ 3 "s_register_operand" "w")) + (match_operand:VDQ 1 "s_register_operand" "0")))] + "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" + "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vmla_ddd") + (const_string "neon_fp_vmla_qqq")) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (if_then_else + (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")) + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_qqq_8_16") + (const_string "neon_mla_qqq_32_qqd_32_scalar")))))] +) + +(define_insn "mul<mode>3neg<mode>add<mode>_neon" + [(set (match_operand:VDQ 0 "s_register_operand" "=w") + (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "0") + (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w") + (match_operand:VDQ 3 "s_register_operand" "w"))))] + "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" + "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vmla_ddd") + (const_string "neon_fp_vmla_qqq")) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (if_then_else + (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")) + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_qqq_8_16") + (const_string "neon_mla_qqq_32_qqd_32_scalar")))))] +) + +(define_insn "ior<mode>3" + [(set (match_operand:VDQ 0 "s_register_operand" "=w,w") + (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0") + (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))] + "TARGET_NEON" +{ + switch (which_alternative) + { + case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2"; + case 1: return neon_output_logic_immediate ("vorr", &operands[2], + <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode)); + default: gcc_unreachable (); + } +} + [(set_attr "neon_type" "neon_int_1")] +) + +(define_insn "iordi3_neon" + [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r") + (ior:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r") + (match_operand:DI 2 "neon_logic_op2" "w,Dl,r,r")))] + "TARGET_NEON" +{ + switch (which_alternative) + { + case 0: return "vorr\t%P0, %P1, %P2"; + case 1: return neon_output_logic_immediate ("vorr", &operands[2], + DImode, 0, VALID_NEON_QREG_MODE (DImode)); + case 2: return "#"; + case 3: return "#"; + default: gcc_unreachable (); + } +} + [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*") + (set_attr "length" "*,*,8,8")] +) + +;; The concrete forms of the Neon immediate-logic instructions are vbic and +;; vorr. We support the pseudo-instruction vand instead, because that +;; corresponds to the canonical form the middle-end expects to use for +;; immediate bitwise-ANDs. + +(define_insn "and<mode>3" + [(set (match_operand:VDQ 0 "s_register_operand" "=w,w") + (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0") + (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))] + "TARGET_NEON" +{ + switch (which_alternative) + { + case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2"; + case 1: return neon_output_logic_immediate ("vand", &operands[2], + <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode)); + default: gcc_unreachable (); + } +} + [(set_attr "neon_type" "neon_int_1")] +) + +(define_insn "anddi3_neon" + [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r") + (and:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r") + (match_operand:DI 2 "neon_inv_logic_op2" "w,DL,r,r")))] + "TARGET_NEON" +{ + switch (which_alternative) + { + case 0: return "vand\t%P0, %P1, %P2"; + case 1: return neon_output_logic_immediate ("vand", &operands[2], + DImode, 1, VALID_NEON_QREG_MODE (DImode)); + case 2: return "#"; + case 3: return "#"; + default: gcc_unreachable (); + } +} + [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*") + (set_attr "length" "*,*,8,8")] +) + +(define_insn "orn<mode>3_neon" + [(set (match_operand:VDQ 0 "s_register_operand" "=w") + (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w") + (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))))] + "TARGET_NEON" + "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set_attr "neon_type" "neon_int_1")] +) + +(define_insn "orndi3_neon" + [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r") + (ior:DI (match_operand:DI 1 "s_register_operand" "w,r,0") + (not:DI (match_operand:DI 2 "s_register_operand" "w,0,r"))))] + "TARGET_NEON" + "@ + vorn\t%P0, %P1, %P2 + # + #" + [(set_attr "neon_type" "neon_int_1,*,*") + (set_attr "length" "*,8,8")] +) + +(define_insn "bic<mode>3_neon" + [(set (match_operand:VDQ 0 "s_register_operand" "=w") + (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w") + (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))))] + "TARGET_NEON" + "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set_attr "neon_type" "neon_int_1")] +) + +;; Compare to *anddi_notdi_di. +(define_insn "bicdi3_neon" + [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r") + (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0")) + (match_operand:DI 1 "s_register_operand" "w,0,r")))] + "TARGET_NEON" + "@ + vbic\t%P0, %P1, %P2 + # + #" + [(set_attr "neon_type" "neon_int_1,*,*") + (set_attr "length" "*,8,8")] +) + +(define_insn "xor<mode>3" + [(set (match_operand:VDQ 0 "s_register_operand" "=w") + (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w") + (match_operand:VDQ 2 "s_register_operand" "w")))] + "TARGET_NEON" + "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set_attr "neon_type" "neon_int_1")] +) + +(define_insn "xordi3_neon" + [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r") + (xor:DI (match_operand:DI 1 "s_register_operand" "%w,0,r") + (match_operand:DI 2 "s_register_operand" "w,r,r")))] + "TARGET_NEON" + "@ + veor\t%P0, %P1, %P2 + # + #" + [(set_attr "neon_type" "neon_int_1,*,*") + (set_attr "length" "*,8,8")] +) + +(define_insn "one_cmpl<mode>2" + [(set (match_operand:VDQ 0 "s_register_operand" "=w") + (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))] + "TARGET_NEON" + "vmvn\t%<V_reg>0, %<V_reg>1" + [(set_attr "neon_type" "neon_int_1")] +) + +(define_insn "abs<mode>2" + [(set (match_operand:VDQW 0 "s_register_operand" "=w") + (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))] + "TARGET_NEON" + "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_3")))] +) + +(define_insn "neg<mode>2" + [(set (match_operand:VDQW 0 "s_register_operand" "=w") + (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))] + "TARGET_NEON" + "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_3")))] +) + +(define_insn "*umin<mode>3_neon" + [(set (match_operand:VDQIW 0 "s_register_operand" "=w") + (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w") + (match_operand:VDQIW 2 "s_register_operand" "w")))] + "TARGET_NEON" + "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set_attr "neon_type" "neon_int_5")] +) + +(define_insn "*umax<mode>3_neon" + [(set (match_operand:VDQIW 0 "s_register_operand" "=w") + (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w") + (match_operand:VDQIW 2 "s_register_operand" "w")))] + "TARGET_NEON" + "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set_attr "neon_type" "neon_int_5")] +) + +(define_insn "*smin<mode>3_neon" + [(set (match_operand:VDQW 0 "s_register_operand" "=w") + (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w") + (match_operand:VDQW 2 "s_register_operand" "w")))] + "TARGET_NEON" + "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_int_5")))] +) + +(define_insn "*smax<mode>3_neon" + [(set (match_operand:VDQW 0 "s_register_operand" "=w") + (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w") + (match_operand:VDQW 2 "s_register_operand" "w")))] + "TARGET_NEON" + "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_int_5")))] +) + +; TODO: V2DI shifts are current disabled because there are bugs in the +; generic vectorizer code. It ends up creating a V2DI constructor with +; SImode elements. + +(define_insn "vashl<mode>3" + [(set (match_operand:VDQIW 0 "s_register_operand" "=w") + (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w") + (match_operand:VDQIW 2 "s_register_operand" "w")))] + "TARGET_NEON" + "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_vshl_ddd") + (const_string "neon_shift_3")))] +) + +; Used for implementing logical shift-right, which is a left-shift by a negative +; amount, with signed operands. This is essentially the same as ashl<mode>3 +; above, but using an unspec in case GCC tries anything tricky with negative +; shift amounts. + +(define_insn "ashl<mode>3_signed" + [(set (match_operand:VDQI 0 "s_register_operand" "=w") + (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w") + (match_operand:VDQI 2 "s_register_operand" "w")] + UNSPEC_ASHIFT_SIGNED))] + "TARGET_NEON" + "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_vshl_ddd") + (const_string "neon_shift_3")))] +) + +; Used for implementing logical shift-right, which is a left-shift by a negative +; amount, with unsigned operands. + +(define_insn "ashl<mode>3_unsigned" + [(set (match_operand:VDQI 0 "s_register_operand" "=w") + (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w") + (match_operand:VDQI 2 "s_register_operand" "w")] + UNSPEC_ASHIFT_UNSIGNED))] + "TARGET_NEON" + "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_vshl_ddd") + (const_string "neon_shift_3")))] +) + +(define_expand "vashr<mode>3" + [(set (match_operand:VDQIW 0 "s_register_operand" "") + (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "") + (match_operand:VDQIW 2 "s_register_operand" "")))] + "TARGET_NEON" +{ + rtx neg = gen_reg_rtx (<MODE>mode); + + emit_insn (gen_neg<mode>2 (neg, operands[2])); + emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg)); + + DONE; +}) + +(define_expand "vlshr<mode>3" + [(set (match_operand:VDQIW 0 "s_register_operand" "") + (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "") + (match_operand:VDQIW 2 "s_register_operand" "")))] + "TARGET_NEON" +{ + rtx neg = gen_reg_rtx (<MODE>mode); + + emit_insn (gen_neg<mode>2 (neg, operands[2])); + emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg)); + + DONE; +}) + +;; Widening operations + +(define_insn "widen_ssum<mode>3" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (plus:<V_widen> (sign_extend:<V_widen> + (match_operand:VW 1 "s_register_operand" "%w")) + (match_operand:<V_widen> 2 "s_register_operand" "w")))] + "TARGET_NEON" + "vaddw.<V_s_elem>\t%q0, %q2, %P1" + [(set_attr "neon_type" "neon_int_3")] +) + +(define_insn "widen_usum<mode>3" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (plus:<V_widen> (zero_extend:<V_widen> + (match_operand:VW 1 "s_register_operand" "%w")) + (match_operand:<V_widen> 2 "s_register_operand" "w")))] + "TARGET_NEON" + "vaddw.<V_u_elem>\t%q0, %q2, %P1" + [(set_attr "neon_type" "neon_int_3")] +) + +;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit +;; shift-count granularity. That's good enough for the middle-end's current +;; needs. + +(define_expand "vec_shr_<mode>" + [(match_operand:VDQ 0 "s_register_operand" "") + (match_operand:VDQ 1 "s_register_operand" "") + (match_operand:SI 2 "const_multiple_of_8_operand" "")] + "TARGET_NEON" +{ + rtx zero_reg; + HOST_WIDE_INT num_bits = INTVAL (operands[2]); + const int width = GET_MODE_BITSIZE (<MODE>mode); + const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode; + rtx (*gen_ext) (rtx, rtx, rtx, rtx) = + (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi; + + if (num_bits == width) + { + emit_move_insn (operands[0], operands[1]); + DONE; + } + + zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode)); + operands[0] = gen_lowpart (bvecmode, operands[0]); + operands[1] = gen_lowpart (bvecmode, operands[1]); + + emit_insn (gen_ext (operands[0], operands[1], zero_reg, + GEN_INT (num_bits / BITS_PER_UNIT))); + DONE; +}) + +(define_expand "vec_shl_<mode>" + [(match_operand:VDQ 0 "s_register_operand" "") + (match_operand:VDQ 1 "s_register_operand" "") + (match_operand:SI 2 "const_multiple_of_8_operand" "")] + "TARGET_NEON" +{ + rtx zero_reg; + HOST_WIDE_INT num_bits = INTVAL (operands[2]); + const int width = GET_MODE_BITSIZE (<MODE>mode); + const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode; + rtx (*gen_ext) (rtx, rtx, rtx, rtx) = + (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi; + + if (num_bits == 0) + { + emit_move_insn (operands[0], CONST0_RTX (<MODE>mode)); + DONE; + } + + num_bits = width - num_bits; + + zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode)); + operands[0] = gen_lowpart (bvecmode, operands[0]); + operands[1] = gen_lowpart (bvecmode, operands[1]); + + emit_insn (gen_ext (operands[0], zero_reg, operands[1], + GEN_INT (num_bits / BITS_PER_UNIT))); + DONE; +}) + +;; Helpers for quad-word reduction operations + +; Add (or smin, smax...) the low N/2 elements of the N-element vector +; operand[1] to the high N/2 elements of same. Put the result in operand[0], an +; N/2-element vector. + +(define_insn "quad_halves_<code>v4si" + [(set (match_operand:V2SI 0 "s_register_operand" "=w") + (vqh_ops:V2SI + (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w") + (parallel [(const_int 0) (const_int 1)])) + (vec_select:V2SI (match_dup 1) + (parallel [(const_int 2) (const_int 3)]))))] + "TARGET_NEON" + "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1" + [(set_attr "vqh_mnem" "<VQH_mnem>") + (set (attr "neon_type") + (if_then_else (eq_attr "vqh_mnem" "vadd") + (const_string "neon_int_1") (const_string "neon_int_5")))] +) + +(define_insn "quad_halves_<code>v4sf" + [(set (match_operand:V2SF 0 "s_register_operand" "=w") + (vqhs_ops:V2SF + (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w") + (parallel [(const_int 0) (const_int 1)])) + (vec_select:V2SF (match_dup 1) + (parallel [(const_int 2) (const_int 3)]))))] + "TARGET_NEON && flag_unsafe_math_optimizations" + "<VQH_mnem>.f32\t%P0, %e1, %f1" + [(set_attr "vqh_mnem" "<VQH_mnem>") + (set (attr "neon_type") + (if_then_else (eq_attr "vqh_mnem" "vadd") + (const_string "neon_int_1") (const_string "neon_int_5")))] +) + +(define_insn "quad_halves_<code>v8hi" + [(set (match_operand:V4HI 0 "s_register_operand" "+w") + (vqh_ops:V4HI + (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w") + (parallel [(const_int 0) (const_int 1) + (const_int 2) (const_int 3)])) + (vec_select:V4HI (match_dup 1) + (parallel [(const_int 4) (const_int 5) + (const_int 6) (const_int 7)]))))] + "TARGET_NEON" + "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1" + [(set_attr "vqh_mnem" "<VQH_mnem>") + (set (attr "neon_type") + (if_then_else (eq_attr "vqh_mnem" "vadd") + (const_string "neon_int_1") (const_string "neon_int_5")))] +) + +(define_insn "quad_halves_<code>v16qi" + [(set (match_operand:V8QI 0 "s_register_operand" "+w") + (vqh_ops:V8QI + (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w") + (parallel [(const_int 0) (const_int 1) + (const_int 2) (const_int 3) + (const_int 4) (const_int 5) + (const_int 6) (const_int 7)])) + (vec_select:V8QI (match_dup 1) + (parallel [(const_int 8) (const_int 9) + (const_int 10) (const_int 11) + (const_int 12) (const_int 13) + (const_int 14) (const_int 15)]))))] + "TARGET_NEON" + "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1" + [(set_attr "vqh_mnem" "<VQH_mnem>") + (set (attr "neon_type") + (if_then_else (eq_attr "vqh_mnem" "vadd") + (const_string "neon_int_1") (const_string "neon_int_5")))] +) + +; FIXME: We wouldn't need the following insns if we could write subregs of +; vector registers. Make an attempt at removing unnecessary moves, though +; we're really at the mercy of the register allocator. + +(define_insn "neon_move_lo_quad_<mode>" + [(set (match_operand:ANY128 0 "s_register_operand" "+w") + (vec_concat:ANY128 + (match_operand:<V_HALF> 1 "s_register_operand" "w") + (vec_select:<V_HALF> + (match_dup 0) + (match_operand:ANY128 2 "vect_par_constant_high" ""))))] + "TARGET_NEON" +{ + int dest = REGNO (operands[0]); + int src = REGNO (operands[1]); + + if (dest != src) + return "vmov\t%e0, %P1"; + else + return ""; +} + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_move_hi_quad_<mode>" + [(set (match_operand:ANY128 0 "s_register_operand" "+w") + (vec_concat:ANY128 + (vec_select:<V_HALF> + (match_dup 0) + (match_operand:ANY128 2 "vect_par_constant_low" "")) + (match_operand:<V_HALF> 1 "s_register_operand" "w")))] + + "TARGET_NEON" +{ + int dest = REGNO (operands[0]); + int src = REGNO (operands[1]); + + if (dest != src) + return "vmov\t%f0, %P1"; + else + return ""; +} + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_expand "move_hi_quad_<mode>" + [(match_operand:ANY128 0 "s_register_operand" "") + (match_operand:<V_HALF> 1 "s_register_operand" "")] + "TARGET_NEON" +{ + rtvec v = rtvec_alloc (<V_mode_nunits>/2); + rtx t1; + int i; + + for (i=0; i < (<V_mode_nunits>/2); i++) + RTVEC_ELT (v, i) = GEN_INT (i); + + t1 = gen_rtx_PARALLEL (<MODE>mode, v); + emit_insn (gen_neon_move_hi_quad_<mode> (operands[0], operands[1], t1)); + + DONE; +}) + +(define_expand "move_lo_quad_<mode>" + [(match_operand:ANY128 0 "s_register_operand" "") + (match_operand:<V_HALF> 1 "s_register_operand" "")] + "TARGET_NEON" +{ + rtvec v = rtvec_alloc (<V_mode_nunits>/2); + rtx t1; + int i; + + for (i=0; i < (<V_mode_nunits>/2); i++) + RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i); + + t1 = gen_rtx_PARALLEL (<MODE>mode, v); + emit_insn (gen_neon_move_lo_quad_<mode> (operands[0], operands[1], t1)); + + DONE; +}) + +;; Reduction operations + +(define_expand "reduc_splus_<mode>" + [(match_operand:VD 0 "s_register_operand" "") + (match_operand:VD 1 "s_register_operand" "")] + "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" +{ + neon_pairwise_reduce (operands[0], operands[1], <MODE>mode, + &gen_neon_vpadd_internal<mode>); + DONE; +}) + +(define_expand "reduc_splus_<mode>" + [(match_operand:VQ 0 "s_register_operand" "") + (match_operand:VQ 1 "s_register_operand" "")] + "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" +{ + rtx step1 = gen_reg_rtx (<V_HALF>mode); + rtx res_d = gen_reg_rtx (<V_HALF>mode); + + emit_insn (gen_quad_halves_plus<mode> (step1, operands[1])); + emit_insn (gen_reduc_splus_<V_half> (res_d, step1)); + emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d)); + + DONE; +}) + +(define_insn "reduc_splus_v2di" + [(set (match_operand:V2DI 0 "s_register_operand" "=w") + (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")] + UNSPEC_VPADD))] + "TARGET_NEON" + "vadd.i64\t%e0, %e1, %f1" + [(set_attr "neon_type" "neon_int_1")] +) + +;; NEON does not distinguish between signed and unsigned addition except on +;; widening operations. +(define_expand "reduc_uplus_<mode>" + [(match_operand:VDQI 0 "s_register_operand" "") + (match_operand:VDQI 1 "s_register_operand" "")] + "TARGET_NEON" +{ + emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1])); + DONE; +}) + +(define_expand "reduc_smin_<mode>" + [(match_operand:VD 0 "s_register_operand" "") + (match_operand:VD 1 "s_register_operand" "")] + "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" +{ + neon_pairwise_reduce (operands[0], operands[1], <MODE>mode, + &gen_neon_vpsmin<mode>); + DONE; +}) + +(define_expand "reduc_smin_<mode>" + [(match_operand:VQ 0 "s_register_operand" "") + (match_operand:VQ 1 "s_register_operand" "")] + "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" +{ + rtx step1 = gen_reg_rtx (<V_HALF>mode); + rtx res_d = gen_reg_rtx (<V_HALF>mode); + + emit_insn (gen_quad_halves_smin<mode> (step1, operands[1])); + emit_insn (gen_reduc_smin_<V_half> (res_d, step1)); + emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d)); + + DONE; +}) + +(define_expand "reduc_smax_<mode>" + [(match_operand:VD 0 "s_register_operand" "") + (match_operand:VD 1 "s_register_operand" "")] + "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" +{ + neon_pairwise_reduce (operands[0], operands[1], <MODE>mode, + &gen_neon_vpsmax<mode>); + DONE; +}) + +(define_expand "reduc_smax_<mode>" + [(match_operand:VQ 0 "s_register_operand" "") + (match_operand:VQ 1 "s_register_operand" "")] + "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" +{ + rtx step1 = gen_reg_rtx (<V_HALF>mode); + rtx res_d = gen_reg_rtx (<V_HALF>mode); + + emit_insn (gen_quad_halves_smax<mode> (step1, operands[1])); + emit_insn (gen_reduc_smax_<V_half> (res_d, step1)); + emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d)); + + DONE; +}) + +(define_expand "reduc_umin_<mode>" + [(match_operand:VDI 0 "s_register_operand" "") + (match_operand:VDI 1 "s_register_operand" "")] + "TARGET_NEON" +{ + neon_pairwise_reduce (operands[0], operands[1], <MODE>mode, + &gen_neon_vpumin<mode>); + DONE; +}) + +(define_expand "reduc_umin_<mode>" + [(match_operand:VQI 0 "s_register_operand" "") + (match_operand:VQI 1 "s_register_operand" "")] + "TARGET_NEON" +{ + rtx step1 = gen_reg_rtx (<V_HALF>mode); + rtx res_d = gen_reg_rtx (<V_HALF>mode); + + emit_insn (gen_quad_halves_umin<mode> (step1, operands[1])); + emit_insn (gen_reduc_umin_<V_half> (res_d, step1)); + emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d)); + + DONE; +}) + +(define_expand "reduc_umax_<mode>" + [(match_operand:VDI 0 "s_register_operand" "") + (match_operand:VDI 1 "s_register_operand" "")] + "TARGET_NEON" +{ + neon_pairwise_reduce (operands[0], operands[1], <MODE>mode, + &gen_neon_vpumax<mode>); + DONE; +}) + +(define_expand "reduc_umax_<mode>" + [(match_operand:VQI 0 "s_register_operand" "") + (match_operand:VQI 1 "s_register_operand" "")] + "TARGET_NEON" +{ + rtx step1 = gen_reg_rtx (<V_HALF>mode); + rtx res_d = gen_reg_rtx (<V_HALF>mode); + + emit_insn (gen_quad_halves_umax<mode> (step1, operands[1])); + emit_insn (gen_reduc_umax_<V_half> (res_d, step1)); + emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d)); + + DONE; +}) + +(define_insn "neon_vpadd_internal<mode>" + [(set (match_operand:VD 0 "s_register_operand" "=w") + (unspec:VD [(match_operand:VD 1 "s_register_operand" "w") + (match_operand:VD 2 "s_register_operand" "w")] + UNSPEC_VPADD))] + "TARGET_NEON" + "vpadd.<V_if_elem>\t%P0, %P1, %P2" + ;; Assume this schedules like vadd. + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_1")))] +) + +(define_insn "neon_vpsmin<mode>" + [(set (match_operand:VD 0 "s_register_operand" "=w") + (unspec:VD [(match_operand:VD 1 "s_register_operand" "w") + (match_operand:VD 2 "s_register_operand" "w")] + UNSPEC_VPSMIN))] + "TARGET_NEON" + "vpmin.<V_s_elem>\t%P0, %P1, %P2" + ;; Assume this schedules like vmin. + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_int_5")))] +) + +(define_insn "neon_vpsmax<mode>" + [(set (match_operand:VD 0 "s_register_operand" "=w") + (unspec:VD [(match_operand:VD 1 "s_register_operand" "w") + (match_operand:VD 2 "s_register_operand" "w")] + UNSPEC_VPSMAX))] + "TARGET_NEON" + "vpmax.<V_s_elem>\t%P0, %P1, %P2" + ;; Assume this schedules like vmax. + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_int_5")))] +) + +(define_insn "neon_vpumin<mode>" + [(set (match_operand:VDI 0 "s_register_operand" "=w") + (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w") + (match_operand:VDI 2 "s_register_operand" "w")] + UNSPEC_VPUMIN))] + "TARGET_NEON" + "vpmin.<V_u_elem>\t%P0, %P1, %P2" + ;; Assume this schedules like umin. + [(set_attr "neon_type" "neon_int_5")] +) + +(define_insn "neon_vpumax<mode>" + [(set (match_operand:VDI 0 "s_register_operand" "=w") + (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w") + (match_operand:VDI 2 "s_register_operand" "w")] + UNSPEC_VPUMAX))] + "TARGET_NEON" + "vpmax.<V_u_elem>\t%P0, %P1, %P2" + ;; Assume this schedules like umax. + [(set_attr "neon_type" "neon_int_5")] +) + +;; Saturating arithmetic + +; NOTE: Neon supports many more saturating variants of instructions than the +; following, but these are all GCC currently understands. +; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself +; yet either, although these patterns may be used by intrinsics when they're +; added. + +(define_insn "*ss_add<mode>_neon" + [(set (match_operand:VD 0 "s_register_operand" "=w") + (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w") + (match_operand:VD 2 "s_register_operand" "w")))] + "TARGET_NEON" + "vqadd.<V_s_elem>\t%P0, %P1, %P2" + [(set_attr "neon_type" "neon_int_4")] +) + +(define_insn "*us_add<mode>_neon" + [(set (match_operand:VD 0 "s_register_operand" "=w") + (us_plus:VD (match_operand:VD 1 "s_register_operand" "w") + (match_operand:VD 2 "s_register_operand" "w")))] + "TARGET_NEON" + "vqadd.<V_u_elem>\t%P0, %P1, %P2" + [(set_attr "neon_type" "neon_int_4")] +) + +(define_insn "*ss_sub<mode>_neon" + [(set (match_operand:VD 0 "s_register_operand" "=w") + (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w") + (match_operand:VD 2 "s_register_operand" "w")))] + "TARGET_NEON" + "vqsub.<V_s_elem>\t%P0, %P1, %P2" + [(set_attr "neon_type" "neon_int_5")] +) + +(define_insn "*us_sub<mode>_neon" + [(set (match_operand:VD 0 "s_register_operand" "=w") + (us_minus:VD (match_operand:VD 1 "s_register_operand" "w") + (match_operand:VD 2 "s_register_operand" "w")))] + "TARGET_NEON" + "vqsub.<V_u_elem>\t%P0, %P1, %P2" + [(set_attr "neon_type" "neon_int_5")] +) + +;; Conditional instructions. These are comparisons with conditional moves for +;; vectors. They perform the assignment: +;; +;; Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2; +;; +;; where op3 is <, <=, ==, !=, >= or >. Operations are performed +;; element-wise. + +(define_expand "vcond<mode>" + [(set (match_operand:VDQW 0 "s_register_operand" "") + (if_then_else:VDQW + (match_operator 3 "arm_comparison_operator" + [(match_operand:VDQW 4 "s_register_operand" "") + (match_operand:VDQW 5 "nonmemory_operand" "")]) + (match_operand:VDQW 1 "s_register_operand" "") + (match_operand:VDQW 2 "s_register_operand" "")))] + "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" +{ + rtx mask; + int inverse = 0, immediate_zero = 0; + /* See the description of "magic" bits in the 'T' case of + arm_print_operand. */ + HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode) + ? 3 : 1; + rtx magic_rtx = GEN_INT (magic_word); + + mask = gen_reg_rtx (<V_cmp_result>mode); + + if (operands[5] == CONST0_RTX (<MODE>mode)) + immediate_zero = 1; + else if (!REG_P (operands[5])) + operands[5] = force_reg (<MODE>mode, operands[5]); + + switch (GET_CODE (operands[3])) + { + case GE: + emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5], + magic_rtx)); + break; + + case GT: + emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5], + magic_rtx)); + break; + + case EQ: + emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5], + magic_rtx)); + break; + + case LE: + if (immediate_zero) + emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5], + magic_rtx)); + else + emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4], + magic_rtx)); + break; + + case LT: + if (immediate_zero) + emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5], + magic_rtx)); + else + emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4], + magic_rtx)); + break; + + case NE: + emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5], + magic_rtx)); + inverse = 1; + break; + + default: + gcc_unreachable (); + } + + if (inverse) + emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2], + operands[1])); + else + emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1], + operands[2])); + + DONE; +}) + +(define_expand "vcondu<mode>" + [(set (match_operand:VDQIW 0 "s_register_operand" "") + (if_then_else:VDQIW + (match_operator 3 "arm_comparison_operator" + [(match_operand:VDQIW 4 "s_register_operand" "") + (match_operand:VDQIW 5 "s_register_operand" "")]) + (match_operand:VDQIW 1 "s_register_operand" "") + (match_operand:VDQIW 2 "s_register_operand" "")))] + "TARGET_NEON" +{ + rtx mask; + int inverse = 0, immediate_zero = 0; + + mask = gen_reg_rtx (<V_cmp_result>mode); + + if (operands[5] == CONST0_RTX (<MODE>mode)) + immediate_zero = 1; + else if (!REG_P (operands[5])) + operands[5] = force_reg (<MODE>mode, operands[5]); + + switch (GET_CODE (operands[3])) + { + case GEU: + emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5], + const0_rtx)); + break; + + case GTU: + emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5], + const0_rtx)); + break; + + case EQ: + emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5], + const0_rtx)); + break; + + case LEU: + if (immediate_zero) + emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5], + const0_rtx)); + else + emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4], + const0_rtx)); + break; + + case LTU: + if (immediate_zero) + emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5], + const0_rtx)); + else + emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4], + const0_rtx)); + break; + + case NE: + emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5], + const0_rtx)); + inverse = 1; + break; + + default: + gcc_unreachable (); + } + + if (inverse) + emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2], + operands[1])); + else + emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1], + operands[2])); + + DONE; +}) + +;; Patterns for builtins. + +; good for plain vadd, vaddq. + +(define_expand "neon_vadd<mode>" + [(match_operand:VDQX 0 "s_register_operand" "=w") + (match_operand:VDQX 1 "s_register_operand" "w") + (match_operand:VDQX 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + "TARGET_NEON" +{ + if (!<Is_float_mode> || flag_unsafe_math_optimizations) + emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2])); + else + emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1], + operands[2])); + DONE; +}) + +; Note that NEON operations don't support the full IEEE 754 standard: in +; particular, denormal values are flushed to zero. This means that GCC cannot +; use those instructions for autovectorization, etc. unless +; -funsafe-math-optimizations is in effect (in which case flush-to-zero +; behaviour is permissible). Intrinsic operations (provided by the arm_neon.h +; header) must work in either case: if -funsafe-math-optimizations is given, +; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics +; expand to unspecs (which may potentially limit the extent to which they might +; be optimized by generic code). + +; Used for intrinsics when flag_unsafe_math_optimizations is false. + +(define_insn "neon_vadd<mode>_unspec" + [(set (match_operand:VDQX 0 "s_register_operand" "=w") + (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w") + (match_operand:VDQX 2 "s_register_operand" "w")] + UNSPEC_VADD))] + "TARGET_NEON" + "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_1")))] +) + +; operand 3 represents in bits: +; bit 0: signed (vs unsigned). +; bit 1: rounding (vs none). + +(define_insn "neon_vaddl<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w") + (match_operand:VDI 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VADDL))] + "TARGET_NEON" + "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2" + [(set_attr "neon_type" "neon_int_3")] +) + +(define_insn "neon_vaddw<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w") + (match_operand:VDI 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VADDW))] + "TARGET_NEON" + "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2" + [(set_attr "neon_type" "neon_int_2")] +) + +; vhadd and vrhadd. + +(define_insn "neon_vhadd<mode>" + [(set (match_operand:VDQIW 0 "s_register_operand" "=w") + (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w") + (match_operand:VDQIW 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VHADD))] + "TARGET_NEON" + "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set_attr "neon_type" "neon_int_4")] +) + +(define_insn "neon_vqadd<mode>" + [(set (match_operand:VDQIX 0 "s_register_operand" "=w") + (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") + (match_operand:VDQIX 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VQADD))] + "TARGET_NEON" + "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set_attr "neon_type" "neon_int_4")] +) + +(define_insn "neon_vaddhn<mode>" + [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") + (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") + (match_operand:VN 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VADDHN))] + "TARGET_NEON" + "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2" + [(set_attr "neon_type" "neon_int_4")] +) + +;; We cannot replace this unspec with mul<mode>3 because of the odd +;; polynomial multiplication case that can specified by operand 3. +(define_insn "neon_vmul<mode>" + [(set (match_operand:VDQW 0 "s_register_operand" "=w") + (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w") + (match_operand:VDQW 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VMUL))] + "TARGET_NEON" + "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (if_then_else + (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long") + (const_string "neon_mul_qqq_8_16_32_ddd_32")) + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_qqq_8_16_32_ddd_32") + (const_string "neon_mul_qqq_8_16_32_ddd_32")))))] +) + +(define_expand "neon_vmla<mode>" + [(match_operand:VDQW 0 "s_register_operand" "=w") + (match_operand:VDQW 1 "s_register_operand" "0") + (match_operand:VDQW 2 "s_register_operand" "w") + (match_operand:VDQW 3 "s_register_operand" "w") + (match_operand:SI 4 "immediate_operand" "i")] + "TARGET_NEON" +{ + if (!<Is_float_mode> || flag_unsafe_math_optimizations) + emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1], + operands[2], operands[3])); + else + emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1], + operands[2], operands[3])); + DONE; +}) + +; Used for intrinsics when flag_unsafe_math_optimizations is false. + +(define_insn "neon_vmla<mode>_unspec" + [(set (match_operand:VDQ 0 "s_register_operand" "=w") + (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0") + (match_operand:VDQ 2 "s_register_operand" "w") + (match_operand:VDQ 3 "s_register_operand" "w")] + UNSPEC_VMLA))] + "TARGET_NEON" + "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vmla_ddd") + (const_string "neon_fp_vmla_qqq")) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (if_then_else + (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")) + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_qqq_8_16") + (const_string "neon_mla_qqq_32_qqd_32_scalar")))))] +) + +(define_insn "neon_vmlal<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") + (match_operand:VW 2 "s_register_operand" "w") + (match_operand:VW 3 "s_register_operand" "w") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_VMLAL))] + "TARGET_NEON" + "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] +) + +(define_expand "neon_vmls<mode>" + [(match_operand:VDQW 0 "s_register_operand" "=w") + (match_operand:VDQW 1 "s_register_operand" "0") + (match_operand:VDQW 2 "s_register_operand" "w") + (match_operand:VDQW 3 "s_register_operand" "w") + (match_operand:SI 4 "immediate_operand" "i")] + "TARGET_NEON" +{ + if (!<Is_float_mode> || flag_unsafe_math_optimizations) + emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0], + operands[1], operands[2], operands[3])); + else + emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1], + operands[2], operands[3])); + DONE; +}) + +; Used for intrinsics when flag_unsafe_math_optimizations is false. + +(define_insn "neon_vmls<mode>_unspec" + [(set (match_operand:VDQ 0 "s_register_operand" "=w") + (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0") + (match_operand:VDQ 2 "s_register_operand" "w") + (match_operand:VDQ 3 "s_register_operand" "w")] + UNSPEC_VMLS))] + "TARGET_NEON" + "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vmla_ddd") + (const_string "neon_fp_vmla_qqq")) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (if_then_else + (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")) + (if_then_else + (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_qqq_8_16") + (const_string "neon_mla_qqq_32_qqd_32_scalar")))))] +) + +(define_insn "neon_vmlsl<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") + (match_operand:VW 2 "s_register_operand" "w") + (match_operand:VW 3 "s_register_operand" "w") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_VMLSL))] + "TARGET_NEON" + "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] +) + +(define_insn "neon_vqdmulh<mode>" + [(set (match_operand:VMDQI 0 "s_register_operand" "=w") + (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w") + (match_operand:VMDQI 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VQDMULH))] + "TARGET_NEON" + "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long") + (const_string "neon_mul_qqq_8_16_32_ddd_32")) + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_qqq_8_16_32_ddd_32") + (const_string "neon_mul_qqq_8_16_32_ddd_32"))))] +) + +(define_insn "neon_vqdmlal<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") + (match_operand:VMDI 2 "s_register_operand" "w") + (match_operand:VMDI 3 "s_register_operand" "w") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_VQDMLAL))] + "TARGET_NEON" + "vqdmlal.<V_s_elem>\t%q0, %P2, %P3" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] +) + +(define_insn "neon_vqdmlsl<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") + (match_operand:VMDI 2 "s_register_operand" "w") + (match_operand:VMDI 3 "s_register_operand" "w") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_VQDMLSL))] + "TARGET_NEON" + "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] +) + +(define_insn "neon_vmull<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w") + (match_operand:VW 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VMULL))] + "TARGET_NEON" + "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long") + (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))] +) + +(define_insn "neon_vqdmull<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w") + (match_operand:VMDI 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VQDMULL))] + "TARGET_NEON" + "vqdmull.<V_s_elem>\t%q0, %P1, %P2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long") + (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))] +) + +(define_expand "neon_vsub<mode>" + [(match_operand:VDQX 0 "s_register_operand" "=w") + (match_operand:VDQX 1 "s_register_operand" "w") + (match_operand:VDQX 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + "TARGET_NEON" +{ + if (!<Is_float_mode> || flag_unsafe_math_optimizations) + emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2])); + else + emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1], + operands[2])); + DONE; +}) + +; Used for intrinsics when flag_unsafe_math_optimizations is false. + +(define_insn "neon_vsub<mode>_unspec" + [(set (match_operand:VDQX 0 "s_register_operand" "=w") + (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w") + (match_operand:VDQX 2 "s_register_operand" "w")] + UNSPEC_VSUB))] + "TARGET_NEON" + "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_2")))] +) + +(define_insn "neon_vsubl<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w") + (match_operand:VDI 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VSUBL))] + "TARGET_NEON" + "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2" + [(set_attr "neon_type" "neon_int_2")] +) + +(define_insn "neon_vsubw<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w") + (match_operand:VDI 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VSUBW))] + "TARGET_NEON" + "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2" + [(set_attr "neon_type" "neon_int_2")] +) + +(define_insn "neon_vqsub<mode>" + [(set (match_operand:VDQIX 0 "s_register_operand" "=w") + (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") + (match_operand:VDQIX 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VQSUB))] + "TARGET_NEON" + "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set_attr "neon_type" "neon_int_5")] +) + +(define_insn "neon_vhsub<mode>" + [(set (match_operand:VDQIW 0 "s_register_operand" "=w") + (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w") + (match_operand:VDQIW 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VHSUB))] + "TARGET_NEON" + "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set_attr "neon_type" "neon_int_5")] +) + +(define_insn "neon_vsubhn<mode>" + [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") + (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") + (match_operand:VN 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VSUBHN))] + "TARGET_NEON" + "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2" + [(set_attr "neon_type" "neon_int_4")] +) + +(define_insn "neon_vceq<mode>" + [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w") + (unspec:<V_cmp_result> + [(match_operand:VDQW 1 "s_register_operand" "w,w") + (match_operand:VDQW 2 "nonmemory_operand" "w,Dz") + (match_operand:SI 3 "immediate_operand" "i,i")] + UNSPEC_VCEQ))] + "TARGET_NEON" + "@ + vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2 + vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_5")))] +) + +(define_insn "neon_vcge<mode>" + [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w") + (unspec:<V_cmp_result> + [(match_operand:VDQW 1 "s_register_operand" "w,w") + (match_operand:VDQW 2 "nonmemory_operand" "w,Dz") + (match_operand:SI 3 "immediate_operand" "i,i")] + UNSPEC_VCGE))] + "TARGET_NEON" + "@ + vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2 + vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_5")))] +) + +(define_insn "neon_vcgt<mode>" + [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w") + (unspec:<V_cmp_result> + [(match_operand:VDQW 1 "s_register_operand" "w,w") + (match_operand:VDQW 2 "nonmemory_operand" "w,Dz") + (match_operand:SI 3 "immediate_operand" "i,i")] + UNSPEC_VCGT))] + "TARGET_NEON" + "@ + vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2 + vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_5")))] +) + +;; VCLE and VCLT only support comparisons with immediate zero (register +;; variants are VCGE and VCGT with operands reversed). + +(define_insn "neon_vcle<mode>" + [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w") + (unspec:<V_cmp_result> + [(match_operand:VDQW 1 "s_register_operand" "w") + (match_operand:VDQW 2 "nonmemory_operand" "Dz") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VCLE))] + "TARGET_NEON" + "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_5")))] +) + +(define_insn "neon_vclt<mode>" + [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w") + (unspec:<V_cmp_result> + [(match_operand:VDQW 1 "s_register_operand" "w") + (match_operand:VDQW 2 "nonmemory_operand" "Dz") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VCLT))] + "TARGET_NEON" + "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_5")))] +) + +(define_insn "neon_vcage<mode>" + [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w") + (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w") + (match_operand:VCVTF 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VCAGE))] + "TARGET_NEON" + "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")))] +) + +(define_insn "neon_vcagt<mode>" + [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w") + (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w") + (match_operand:VCVTF 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VCAGT))] + "TARGET_NEON" + "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")))] +) + +(define_insn "neon_vtst<mode>" + [(set (match_operand:VDQIW 0 "s_register_operand" "=w") + (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w") + (match_operand:VDQIW 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VTST))] + "TARGET_NEON" + "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set_attr "neon_type" "neon_int_4")] +) + +(define_insn "neon_vabd<mode>" + [(set (match_operand:VDQW 0 "s_register_operand" "=w") + (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w") + (match_operand:VDQW 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VABD))] + "TARGET_NEON" + "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_5")))] +) + +(define_insn "neon_vabdl<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w") + (match_operand:VW 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VABDL))] + "TARGET_NEON" + "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2" + [(set_attr "neon_type" "neon_int_5")] +) + +(define_insn "neon_vaba<mode>" + [(set (match_operand:VDQIW 0 "s_register_operand" "=w") + (plus:VDQIW (match_operand:VDQIW 1 "s_register_operand" "0") + (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w") + (match_operand:VDQIW 3 "s_register_operand" "w") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_VABD)))] + "TARGET_NEON" + "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_vaba") (const_string "neon_vaba_qqq")))] +) + +(define_insn "neon_vabal<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (plus:<V_widen> (match_operand:<V_widen> 1 "s_register_operand" "0") + (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w") + (match_operand:VW 3 "s_register_operand" "w") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_VABDL)))] + "TARGET_NEON" + "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3" + [(set_attr "neon_type" "neon_vaba")] +) + +(define_insn "neon_vmax<mode>" + [(set (match_operand:VDQW 0 "s_register_operand" "=w") + (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w") + (match_operand:VDQW 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VMAX))] + "TARGET_NEON" + "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_5")))] +) + +(define_insn "neon_vmin<mode>" + [(set (match_operand:VDQW 0 "s_register_operand" "=w") + (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w") + (match_operand:VDQW 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VMIN))] + "TARGET_NEON" + "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")) + (const_string "neon_int_5")))] +) + +(define_expand "neon_vpadd<mode>" + [(match_operand:VD 0 "s_register_operand" "=w") + (match_operand:VD 1 "s_register_operand" "w") + (match_operand:VD 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + "TARGET_NEON" +{ + emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1], + operands[2])); + DONE; +}) + +(define_insn "neon_vpaddl<mode>" + [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w") + (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VPADDL))] + "TARGET_NEON" + "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1" + ;; Assume this schedules like vaddl. + [(set_attr "neon_type" "neon_int_3")] +) + +(define_insn "neon_vpadal<mode>" + [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w") + (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0") + (match_operand:VDQIW 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VPADAL))] + "TARGET_NEON" + "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2" + ;; Assume this schedules like vpadd. + [(set_attr "neon_type" "neon_int_1")] +) + +(define_insn "neon_vpmax<mode>" + [(set (match_operand:VD 0 "s_register_operand" "=w") + (unspec:VD [(match_operand:VD 1 "s_register_operand" "w") + (match_operand:VD 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VPMAX))] + "TARGET_NEON" + "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + ;; Assume this schedules like vmax. + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_int_5")))] +) + +(define_insn "neon_vpmin<mode>" + [(set (match_operand:VD 0 "s_register_operand" "=w") + (unspec:VD [(match_operand:VD 1 "s_register_operand" "w") + (match_operand:VD 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VPMIN))] + "TARGET_NEON" + "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + ;; Assume this schedules like vmin. + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_int_5")))] +) + +(define_insn "neon_vrecps<mode>" + [(set (match_operand:VCVTF 0 "s_register_operand" "=w") + (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w") + (match_operand:VCVTF 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VRECPS))] + "TARGET_NEON" + "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vrecps_vrsqrts_ddd") + (const_string "neon_fp_vrecps_vrsqrts_qqq")))] +) + +(define_insn "neon_vrsqrts<mode>" + [(set (match_operand:VCVTF 0 "s_register_operand" "=w") + (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w") + (match_operand:VCVTF 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VRSQRTS))] + "TARGET_NEON" + "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vrecps_vrsqrts_ddd") + (const_string "neon_fp_vrecps_vrsqrts_qqq")))] +) + +(define_expand "neon_vabs<mode>" + [(match_operand:VDQW 0 "s_register_operand" "") + (match_operand:VDQW 1 "s_register_operand" "") + (match_operand:SI 2 "immediate_operand" "")] + "TARGET_NEON" +{ + emit_insn (gen_abs<mode>2 (operands[0], operands[1])); + DONE; +}) + +(define_insn "neon_vqabs<mode>" + [(set (match_operand:VDQIW 0 "s_register_operand" "=w") + (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VQABS))] + "TARGET_NEON" + "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1" + [(set_attr "neon_type" "neon_vqneg_vqabs")] +) + +(define_expand "neon_vneg<mode>" + [(match_operand:VDQW 0 "s_register_operand" "") + (match_operand:VDQW 1 "s_register_operand" "") + (match_operand:SI 2 "immediate_operand" "")] + "TARGET_NEON" +{ + emit_insn (gen_neg<mode>2 (operands[0], operands[1])); + DONE; +}) + +(define_insn "neon_vqneg<mode>" + [(set (match_operand:VDQIW 0 "s_register_operand" "=w") + (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VQNEG))] + "TARGET_NEON" + "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1" + [(set_attr "neon_type" "neon_vqneg_vqabs")] +) + +(define_insn "neon_vcls<mode>" + [(set (match_operand:VDQIW 0 "s_register_operand" "=w") + (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VCLS))] + "TARGET_NEON" + "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1" + [(set_attr "neon_type" "neon_int_1")] +) + +(define_insn "clz<mode>2" + [(set (match_operand:VDQIW 0 "s_register_operand" "=w") + (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))] + "TARGET_NEON" + "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1" + [(set_attr "neon_type" "neon_int_1")] +) + +(define_expand "neon_vclz<mode>" + [(match_operand:VDQIW 0 "s_register_operand" "") + (match_operand:VDQIW 1 "s_register_operand" "") + (match_operand:SI 2 "immediate_operand" "")] + "TARGET_NEON" +{ + emit_insn (gen_clz<mode>2 (operands[0], operands[1])); + DONE; +}) + +(define_insn "popcount<mode>2" + [(set (match_operand:VE 0 "s_register_operand" "=w") + (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))] + "TARGET_NEON" + "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1" + [(set_attr "neon_type" "neon_int_1")] +) + +(define_expand "neon_vcnt<mode>" + [(match_operand:VE 0 "s_register_operand" "=w") + (match_operand:VE 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + "TARGET_NEON" +{ + emit_insn (gen_popcount<mode>2 (operands[0], operands[1])); + DONE; +}) + +(define_insn "neon_vrecpe<mode>" + [(set (match_operand:V32 0 "s_register_operand" "=w") + (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VRECPE))] + "TARGET_NEON" + "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")))] +) + +(define_insn "neon_vrsqrte<mode>" + [(set (match_operand:V32 0 "s_register_operand" "=w") + (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VRSQRTE))] + "TARGET_NEON" + "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")))] +) + +(define_expand "neon_vmvn<mode>" + [(match_operand:VDQIW 0 "s_register_operand" "") + (match_operand:VDQIW 1 "s_register_operand" "") + (match_operand:SI 2 "immediate_operand" "")] + "TARGET_NEON" +{ + emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1])); + DONE; +}) + +(define_insn "neon_vget_lane<mode>_sext_internal" + [(set (match_operand:SI 0 "s_register_operand" "=r") + (sign_extend:SI + (vec_select:<V_elem> + (match_operand:VD 1 "s_register_operand" "w") + (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))] + "TARGET_NEON" +{ + if (BYTES_BIG_ENDIAN) + { + int elt = INTVAL (operands[2]); + elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt; + operands[2] = GEN_INT (elt); + } + return "vmov%?.s<V_sz_elem>\t%0, %P1[%c2]"; +} + [(set_attr "predicable" "yes") + (set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vget_lane<mode>_zext_internal" + [(set (match_operand:SI 0 "s_register_operand" "=r") + (zero_extend:SI + (vec_select:<V_elem> + (match_operand:VD 1 "s_register_operand" "w") + (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))] + "TARGET_NEON" +{ + if (BYTES_BIG_ENDIAN) + { + int elt = INTVAL (operands[2]); + elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt; + operands[2] = GEN_INT (elt); + } + return "vmov%?.u<V_sz_elem>\t%0, %P1[%c2]"; +} + [(set_attr "predicable" "yes") + (set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vget_lane<mode>_sext_internal" + [(set (match_operand:SI 0 "s_register_operand" "=r") + (sign_extend:SI + (vec_select:<V_elem> + (match_operand:VQ 1 "s_register_operand" "w") + (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))] + "TARGET_NEON" +{ + rtx ops[3]; + int regno = REGNO (operands[1]); + unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2; + unsigned int elt = INTVAL (operands[2]); + unsigned int elt_adj = elt % halfelts; + + if (BYTES_BIG_ENDIAN) + elt_adj = halfelts - 1 - elt_adj; + + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts)); + ops[2] = GEN_INT (elt_adj); + output_asm_insn ("vmov%?.s<V_sz_elem>\t%0, %P1[%c2]", ops); + + return ""; +} + [(set_attr "predicable" "yes") + (set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vget_lane<mode>_zext_internal" + [(set (match_operand:SI 0 "s_register_operand" "=r") + (zero_extend:SI + (vec_select:<V_elem> + (match_operand:VQ 1 "s_register_operand" "w") + (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))] + "TARGET_NEON" +{ + rtx ops[3]; + int regno = REGNO (operands[1]); + unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2; + unsigned int elt = INTVAL (operands[2]); + unsigned int elt_adj = elt % halfelts; + + if (BYTES_BIG_ENDIAN) + elt_adj = halfelts - 1 - elt_adj; + + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts)); + ops[2] = GEN_INT (elt_adj); + output_asm_insn ("vmov%?.u<V_sz_elem>\t%0, %P1[%c2]", ops); + + return ""; +} + [(set_attr "predicable" "yes") + (set_attr "neon_type" "neon_bp_simple")] +) + +(define_expand "neon_vget_lane<mode>" + [(match_operand:<V_ext> 0 "s_register_operand" "") + (match_operand:VDQW 1 "s_register_operand" "") + (match_operand:SI 2 "immediate_operand" "") + (match_operand:SI 3 "immediate_operand" "")] + "TARGET_NEON" +{ + HOST_WIDE_INT magic = INTVAL (operands[3]); + rtx insn; + + neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode)); + + if (BYTES_BIG_ENDIAN) + { + /* The intrinsics are defined in terms of a model where the + element ordering in memory is vldm order, whereas the generic + RTL is defined in terms of a model where the element ordering + in memory is array order. Convert the lane number to conform + to this model. */ + unsigned int elt = INTVAL (operands[2]); + unsigned int reg_nelts + = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)); + elt ^= reg_nelts - 1; + operands[2] = GEN_INT (elt); + } + + if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32) + insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]); + else + { + if ((magic & 1) != 0) + insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1], + operands[2]); + else + insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1], + operands[2]); + } + emit_insn (insn); + DONE; +}) + +; Operand 3 (info word) is ignored because it does nothing useful with 64-bit +; elements. + +(define_expand "neon_vget_lanedi" + [(match_operand:DI 0 "s_register_operand" "=r") + (match_operand:DI 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (match_operand:SI 3 "immediate_operand" "i")] + "TARGET_NEON" +{ + neon_lane_bounds (operands[2], 0, 1); + emit_move_insn (operands[0], operands[1]); + DONE; +}) + +(define_expand "neon_vget_lanev2di" + [(match_operand:DI 0 "s_register_operand" "=r") + (match_operand:V2DI 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (match_operand:SI 3 "immediate_operand" "i")] + "TARGET_NEON" +{ + neon_lane_bounds (operands[2], 0, 2); + emit_insn (gen_vec_extractv2di (operands[0], operands[1], operands[2])); + DONE; +}) + +(define_expand "neon_vset_lane<mode>" + [(match_operand:VDQ 0 "s_register_operand" "=w") + (match_operand:<V_elem> 1 "s_register_operand" "r") + (match_operand:VDQ 2 "s_register_operand" "0") + (match_operand:SI 3 "immediate_operand" "i")] + "TARGET_NEON" +{ + unsigned int elt = INTVAL (operands[3]); + neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); + + if (BYTES_BIG_ENDIAN) + { + unsigned int reg_nelts + = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)); + elt ^= reg_nelts - 1; + } + + emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1], + GEN_INT (1 << elt), operands[2])); + DONE; +}) + +; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored. + +(define_expand "neon_vset_lanedi" + [(match_operand:DI 0 "s_register_operand" "=w") + (match_operand:DI 1 "s_register_operand" "r") + (match_operand:DI 2 "s_register_operand" "0") + (match_operand:SI 3 "immediate_operand" "i")] + "TARGET_NEON" +{ + neon_lane_bounds (operands[3], 0, 1); + emit_move_insn (operands[0], operands[1]); + DONE; +}) + +(define_expand "neon_vcreate<mode>" + [(match_operand:VDX 0 "s_register_operand" "") + (match_operand:DI 1 "general_operand" "")] + "TARGET_NEON" +{ + rtx src = gen_lowpart (<MODE>mode, operands[1]); + emit_move_insn (operands[0], src); + DONE; +}) + +(define_insn "neon_vdup_n<mode>" + [(set (match_operand:VX 0 "s_register_operand" "=w") + (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))] + "TARGET_NEON" + "vdup%?.<V_sz_elem>\t%<V_reg>0, %1" + ;; Assume this schedules like vmov. + [(set_attr "predicable" "yes") + (set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vdup_n<mode>" + [(set (match_operand:V32 0 "s_register_operand" "=w,w") + (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))] + "TARGET_NEON" + "@ + vdup%?.<V_sz_elem>\t%<V_reg>0, %1 + vdup%?.<V_sz_elem>\t%<V_reg>0, %y1" + ;; Assume this schedules like vmov. + [(set_attr "predicable" "yes") + (set_attr "neon_type" "neon_bp_simple")] +) + +(define_expand "neon_vdup_ndi" + [(match_operand:DI 0 "s_register_operand" "=w") + (match_operand:DI 1 "s_register_operand" "r")] + "TARGET_NEON" +{ + emit_move_insn (operands[0], operands[1]); + DONE; +} +) + +(define_insn "neon_vdup_nv2di" + [(set (match_operand:V2DI 0 "s_register_operand" "=w,w") + (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))] + "TARGET_NEON" + "@ + vmov%?\t%e0, %Q1, %R1\;vmov%?\t%f0, %Q1, %R1 + vmov%?\t%e0, %P1\;vmov%?\t%f0, %P1" + [(set_attr "predicable" "yes") + (set_attr "length" "8") + (set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vdup_lane<mode>_internal" + [(set (match_operand:VDQW 0 "s_register_operand" "=w") + (vec_duplicate:VDQW + (vec_select:<V_elem> + (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w") + (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))] + "TARGET_NEON" +{ + if (BYTES_BIG_ENDIAN) + { + int elt = INTVAL (operands[2]); + elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt; + operands[2] = GEN_INT (elt); + } + if (<Is_d_reg>) + return "vdup.<V_sz_elem>\t%P0, %P1[%c2]"; + else + return "vdup.<V_sz_elem>\t%q0, %P1[%c2]"; +} + ;; Assume this schedules like vmov. + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_expand "neon_vdup_lane<mode>" + [(match_operand:VDQW 0 "s_register_operand" "=w") + (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + "TARGET_NEON" +{ + neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode)); + if (BYTES_BIG_ENDIAN) + { + unsigned int elt = INTVAL (operands[2]); + unsigned int reg_nelts + = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode)); + elt ^= reg_nelts - 1; + operands[2] = GEN_INT (elt); + } + emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1], + operands[2])); + DONE; +}) + +; Scalar index is ignored, since only zero is valid here. +(define_expand "neon_vdup_lanedi" + [(match_operand:DI 0 "s_register_operand" "=w") + (match_operand:DI 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + "TARGET_NEON" +{ + neon_lane_bounds (operands[2], 0, 1); + emit_move_insn (operands[0], operands[1]); + DONE; +}) + +; Likewise for v2di, as the DImode second operand has only a single element. +(define_expand "neon_vdup_lanev2di" + [(match_operand:V2DI 0 "s_register_operand" "=w") + (match_operand:DI 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + "TARGET_NEON" +{ + neon_lane_bounds (operands[2], 0, 1); + emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1])); + DONE; +}) + +;; In this insn, operand 1 should be low, and operand 2 the high part of the +;; dest vector. +;; FIXME: A different implementation of this builtin could make it much +;; more likely that we wouldn't actually need to output anything (we could make +;; it so that the reg allocator puts things in the right places magically +;; instead). Lack of subregs for vectors makes that tricky though, I think. + +(define_insn "neon_vcombine<mode>" + [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w") + (vec_concat:<V_DOUBLE> (match_operand:VDX 1 "s_register_operand" "w") + (match_operand:VDX 2 "s_register_operand" "w")))] + "TARGET_NEON" +{ + int dest = REGNO (operands[0]); + int src1 = REGNO (operands[1]); + int src2 = REGNO (operands[2]); + rtx destlo; + + if (src1 == dest && src2 == dest + 2) + return ""; + else if (src2 == dest && src1 == dest + 2) + /* Special case of reversed high/low parts. */ + return "vswp\t%P1, %P2"; + + destlo = gen_rtx_REG (<MODE>mode, dest); + + if (!reg_overlap_mentioned_p (operands[2], destlo)) + { + /* Try to avoid unnecessary moves if part of the result is in the right + place already. */ + if (src1 != dest) + output_asm_insn ("vmov\t%e0, %P1", operands); + if (src2 != dest + 2) + output_asm_insn ("vmov\t%f0, %P2", operands); + } + else + { + if (src2 != dest + 2) + output_asm_insn ("vmov\t%f0, %P2", operands); + if (src1 != dest) + output_asm_insn ("vmov\t%e0, %P1", operands); + } + + return ""; +} + ;; We set the neon_type attribute based on the vmov instructions above. + [(set_attr "length" "8") + (set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vget_highv16qi" + [(set (match_operand:V8QI 0 "s_register_operand" "=w") + (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w") + (parallel [(const_int 8) (const_int 9) + (const_int 10) (const_int 11) + (const_int 12) (const_int 13) + (const_int 14) (const_int 15)])))] + "TARGET_NEON" +{ + int dest = REGNO (operands[0]); + int src = REGNO (operands[1]); + + if (dest != src + 2) + return "vmov\t%P0, %f1"; + else + return ""; +} + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vget_highv8hi" + [(set (match_operand:V4HI 0 "s_register_operand" "=w") + (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w") + (parallel [(const_int 4) (const_int 5) + (const_int 6) (const_int 7)])))] + "TARGET_NEON" +{ + int dest = REGNO (operands[0]); + int src = REGNO (operands[1]); + + if (dest != src + 2) + return "vmov\t%P0, %f1"; + else + return ""; +} + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vget_highv4si" + [(set (match_operand:V2SI 0 "s_register_operand" "=w") + (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w") + (parallel [(const_int 2) (const_int 3)])))] + "TARGET_NEON" +{ + int dest = REGNO (operands[0]); + int src = REGNO (operands[1]); + + if (dest != src + 2) + return "vmov\t%P0, %f1"; + else + return ""; +} + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vget_highv4sf" + [(set (match_operand:V2SF 0 "s_register_operand" "=w") + (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w") + (parallel [(const_int 2) (const_int 3)])))] + "TARGET_NEON" +{ + int dest = REGNO (operands[0]); + int src = REGNO (operands[1]); + + if (dest != src + 2) + return "vmov\t%P0, %f1"; + else + return ""; +} + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vget_highv2di" + [(set (match_operand:DI 0 "s_register_operand" "=w") + (vec_select:DI (match_operand:V2DI 1 "s_register_operand" "w") + (parallel [(const_int 1)])))] + "TARGET_NEON" +{ + int dest = REGNO (operands[0]); + int src = REGNO (operands[1]); + + if (dest != src + 2) + return "vmov\t%P0, %f1"; + else + return ""; +} + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vget_lowv16qi" + [(set (match_operand:V8QI 0 "s_register_operand" "=w") + (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w") + (parallel [(const_int 0) (const_int 1) + (const_int 2) (const_int 3) + (const_int 4) (const_int 5) + (const_int 6) (const_int 7)])))] + "TARGET_NEON" +{ + int dest = REGNO (operands[0]); + int src = REGNO (operands[1]); + + if (dest != src) + return "vmov\t%P0, %e1"; + else + return ""; +} + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vget_lowv8hi" + [(set (match_operand:V4HI 0 "s_register_operand" "=w") + (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w") + (parallel [(const_int 0) (const_int 1) + (const_int 2) (const_int 3)])))] + "TARGET_NEON" +{ + int dest = REGNO (operands[0]); + int src = REGNO (operands[1]); + + if (dest != src) + return "vmov\t%P0, %e1"; + else + return ""; +} + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vget_lowv4si" + [(set (match_operand:V2SI 0 "s_register_operand" "=w") + (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w") + (parallel [(const_int 0) (const_int 1)])))] + "TARGET_NEON" +{ + int dest = REGNO (operands[0]); + int src = REGNO (operands[1]); + + if (dest != src) + return "vmov\t%P0, %e1"; + else + return ""; +} + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vget_lowv4sf" + [(set (match_operand:V2SF 0 "s_register_operand" "=w") + (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w") + (parallel [(const_int 0) (const_int 1)])))] + "TARGET_NEON" +{ + int dest = REGNO (operands[0]); + int src = REGNO (operands[1]); + + if (dest != src) + return "vmov\t%P0, %e1"; + else + return ""; +} + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vget_lowv2di" + [(set (match_operand:DI 0 "s_register_operand" "=w") + (vec_select:DI (match_operand:V2DI 1 "s_register_operand" "w") + (parallel [(const_int 0)])))] + "TARGET_NEON" +{ + int dest = REGNO (operands[0]); + int src = REGNO (operands[1]); + + if (dest != src) + return "vmov\t%P0, %e1"; + else + return ""; +} + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vcvt<mode>" + [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w") + (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VCVT))] + "TARGET_NEON" + "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")))] +) + +(define_insn "neon_vcvt<mode>" + [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w") + (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VCVT))] + "TARGET_NEON" + "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")))] +) + +(define_insn "neon_vcvt_n<mode>" + [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w") + (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VCVT_N))] + "TARGET_NEON" +{ + neon_const_bounds (operands[2], 1, 33); + return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")))] +) + +(define_insn "neon_vcvt_n<mode>" + [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w") + (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VCVT_N))] + "TARGET_NEON" +{ + neon_const_bounds (operands[2], 1, 33); + return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_fp_vadd_ddd_vabs_dd") + (const_string "neon_fp_vadd_qqq_vabs_qq")))] +) + +(define_insn "neon_vmovn<mode>" + [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") + (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VMOVN))] + "TARGET_NEON" + "vmovn.<V_if_elem>\t%P0, %q1" + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vqmovn<mode>" + [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") + (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VQMOVN))] + "TARGET_NEON" + "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1" + [(set_attr "neon_type" "neon_shift_2")] +) + +(define_insn "neon_vqmovun<mode>" + [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") + (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VQMOVUN))] + "TARGET_NEON" + "vqmovun.<V_s_elem>\t%P0, %q1" + [(set_attr "neon_type" "neon_shift_2")] +) + +(define_insn "neon_vmovl<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VMOVL))] + "TARGET_NEON" + "vmovl.%T2%#<V_sz_elem>\t%q0, %P1" + [(set_attr "neon_type" "neon_shift_1")] +) + +(define_insn "neon_vmul_lane<mode>" + [(set (match_operand:VMD 0 "s_register_operand" "=w") + (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w") + (match_operand:VMD 2 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 3 "immediate_operand" "i") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_VMUL_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (const_string "neon_fp_vmul_ddd") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar") + (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar"))))] +) + +(define_insn "neon_vmul_lane<mode>" + [(set (match_operand:VMQ 0 "s_register_operand" "=w") + (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w") + (match_operand:<V_HALF> 2 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 3 "immediate_operand" "i") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_VMUL_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode)); + return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (const_string "neon_fp_vmul_qqd") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar") + (const_string "neon_mul_qqd_32_scalar"))))] +) + +(define_insn "neon_vmull_lane<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w") + (match_operand:VMDI 2 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 3 "immediate_operand" "i") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_VMULL_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar") + (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))] +) + +(define_insn "neon_vqdmull_lane<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w") + (match_operand:VMDI 2 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 3 "immediate_operand" "i") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_VQDMULL_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar") + (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))] +) + +(define_insn "neon_vqdmulh_lane<mode>" + [(set (match_operand:VMQI 0 "s_register_operand" "=w") + (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w") + (match_operand:<V_HALF> 2 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 3 "immediate_operand" "i") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_VQDMULH_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar") + (const_string "neon_mul_qqd_32_scalar")))] +) + +(define_insn "neon_vqdmulh_lane<mode>" + [(set (match_operand:VMDI 0 "s_register_operand" "=w") + (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w") + (match_operand:VMDI 2 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 3 "immediate_operand" "i") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_VQDMULH_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar") + (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))] +) + +(define_insn "neon_vmla_lane<mode>" + [(set (match_operand:VMD 0 "s_register_operand" "=w") + (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0") + (match_operand:VMD 2 "s_register_operand" "w") + (match_operand:VMD 3 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 4 "immediate_operand" "i") + (match_operand:SI 5 "immediate_operand" "i")] + UNSPEC_VMLA_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (const_string "neon_fp_vmla_ddd_scalar") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))] +) + +(define_insn "neon_vmla_lane<mode>" + [(set (match_operand:VMQ 0 "s_register_operand" "=w") + (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0") + (match_operand:VMQ 2 "s_register_operand" "w") + (match_operand:<V_HALF> 3 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 4 "immediate_operand" "i") + (match_operand:SI 5 "immediate_operand" "i")] + UNSPEC_VMLA_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (const_string "neon_fp_vmla_qqq_scalar") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long") + (const_string "neon_mla_qqq_32_qqd_32_scalar"))))] +) + +(define_insn "neon_vmlal_lane<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") + (match_operand:VMDI 2 "s_register_operand" "w") + (match_operand:VMDI 3 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 4 "immediate_operand" "i") + (match_operand:SI 5 "immediate_operand" "i")] + UNSPEC_VMLAL_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] +) + +(define_insn "neon_vqdmlal_lane<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") + (match_operand:VMDI 2 "s_register_operand" "w") + (match_operand:VMDI 3 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 4 "immediate_operand" "i") + (match_operand:SI 5 "immediate_operand" "i")] + UNSPEC_VQDMLAL_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] +) + +(define_insn "neon_vmls_lane<mode>" + [(set (match_operand:VMD 0 "s_register_operand" "=w") + (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0") + (match_operand:VMD 2 "s_register_operand" "w") + (match_operand:VMD 3 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 4 "immediate_operand" "i") + (match_operand:SI 5 "immediate_operand" "i")] + UNSPEC_VMLS_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (const_string "neon_fp_vmla_ddd_scalar") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))] +) + +(define_insn "neon_vmls_lane<mode>" + [(set (match_operand:VMQ 0 "s_register_operand" "=w") + (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0") + (match_operand:VMQ 2 "s_register_operand" "w") + (match_operand:<V_HALF> 3 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 4 "immediate_operand" "i") + (match_operand:SI 5 "immediate_operand" "i")] + UNSPEC_VMLS_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) + (const_string "neon_fp_vmla_qqq_scalar") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long") + (const_string "neon_mla_qqq_32_qqd_32_scalar"))))] +) + +(define_insn "neon_vmlsl_lane<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") + (match_operand:VMDI 2 "s_register_operand" "w") + (match_operand:VMDI 3 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 4 "immediate_operand" "i") + (match_operand:SI 5 "immediate_operand" "i")] + UNSPEC_VMLSL_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] +) + +(define_insn "neon_vqdmlsl_lane<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") + (match_operand:VMDI 2 "s_register_operand" "w") + (match_operand:VMDI 3 "s_register_operand" + "<scalar_mul_constraint>") + (match_operand:SI 4 "immediate_operand" "i") + (match_operand:SI 5 "immediate_operand" "i")] + UNSPEC_VQDMLSL_LANE))] + "TARGET_NEON" +{ + neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0)) + (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar") + (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] +) + +; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a +; core register into a temp register, then use a scalar taken from that. This +; isn't an optimal solution if e.g. the scalar has just been read from memory +; or extracted from another vector. The latter case it's currently better to +; use the "_lane" variant, and the former case can probably be implemented +; using vld1_lane, but that hasn't been done yet. + +(define_expand "neon_vmul_n<mode>" + [(match_operand:VMD 0 "s_register_operand" "") + (match_operand:VMD 1 "s_register_operand" "") + (match_operand:<V_elem> 2 "s_register_operand" "") + (match_operand:SI 3 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<MODE>mode); + emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx)); + emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp, + const0_rtx, const0_rtx)); + DONE; +}) + +(define_expand "neon_vmul_n<mode>" + [(match_operand:VMQ 0 "s_register_operand" "") + (match_operand:VMQ 1 "s_register_operand" "") + (match_operand:<V_elem> 2 "s_register_operand" "") + (match_operand:SI 3 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<V_HALF>mode); + emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx)); + emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp, + const0_rtx, const0_rtx)); + DONE; +}) + +(define_expand "neon_vmull_n<mode>" + [(match_operand:<V_widen> 0 "s_register_operand" "") + (match_operand:VMDI 1 "s_register_operand" "") + (match_operand:<V_elem> 2 "s_register_operand" "") + (match_operand:SI 3 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<MODE>mode); + emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx)); + emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp, + const0_rtx, operands[3])); + DONE; +}) + +(define_expand "neon_vqdmull_n<mode>" + [(match_operand:<V_widen> 0 "s_register_operand" "") + (match_operand:VMDI 1 "s_register_operand" "") + (match_operand:<V_elem> 2 "s_register_operand" "") + (match_operand:SI 3 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<MODE>mode); + emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx)); + emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp, + const0_rtx, const0_rtx)); + DONE; +}) + +(define_expand "neon_vqdmulh_n<mode>" + [(match_operand:VMDI 0 "s_register_operand" "") + (match_operand:VMDI 1 "s_register_operand" "") + (match_operand:<V_elem> 2 "s_register_operand" "") + (match_operand:SI 3 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<MODE>mode); + emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx)); + emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp, + const0_rtx, operands[3])); + DONE; +}) + +(define_expand "neon_vqdmulh_n<mode>" + [(match_operand:VMQI 0 "s_register_operand" "") + (match_operand:VMQI 1 "s_register_operand" "") + (match_operand:<V_elem> 2 "s_register_operand" "") + (match_operand:SI 3 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<V_HALF>mode); + emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx)); + emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp, + const0_rtx, operands[3])); + DONE; +}) + +(define_expand "neon_vmla_n<mode>" + [(match_operand:VMD 0 "s_register_operand" "") + (match_operand:VMD 1 "s_register_operand" "") + (match_operand:VMD 2 "s_register_operand" "") + (match_operand:<V_elem> 3 "s_register_operand" "") + (match_operand:SI 4 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<MODE>mode); + emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx)); + emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2], + tmp, const0_rtx, operands[4])); + DONE; +}) + +(define_expand "neon_vmla_n<mode>" + [(match_operand:VMQ 0 "s_register_operand" "") + (match_operand:VMQ 1 "s_register_operand" "") + (match_operand:VMQ 2 "s_register_operand" "") + (match_operand:<V_elem> 3 "s_register_operand" "") + (match_operand:SI 4 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<V_HALF>mode); + emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx)); + emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2], + tmp, const0_rtx, operands[4])); + DONE; +}) + +(define_expand "neon_vmlal_n<mode>" + [(match_operand:<V_widen> 0 "s_register_operand" "") + (match_operand:<V_widen> 1 "s_register_operand" "") + (match_operand:VMDI 2 "s_register_operand" "") + (match_operand:<V_elem> 3 "s_register_operand" "") + (match_operand:SI 4 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<MODE>mode); + emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx)); + emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2], + tmp, const0_rtx, operands[4])); + DONE; +}) + +(define_expand "neon_vqdmlal_n<mode>" + [(match_operand:<V_widen> 0 "s_register_operand" "") + (match_operand:<V_widen> 1 "s_register_operand" "") + (match_operand:VMDI 2 "s_register_operand" "") + (match_operand:<V_elem> 3 "s_register_operand" "") + (match_operand:SI 4 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<MODE>mode); + emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx)); + emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2], + tmp, const0_rtx, operands[4])); + DONE; +}) + +(define_expand "neon_vmls_n<mode>" + [(match_operand:VMD 0 "s_register_operand" "") + (match_operand:VMD 1 "s_register_operand" "") + (match_operand:VMD 2 "s_register_operand" "") + (match_operand:<V_elem> 3 "s_register_operand" "") + (match_operand:SI 4 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<MODE>mode); + emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx)); + emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2], + tmp, const0_rtx, operands[4])); + DONE; +}) + +(define_expand "neon_vmls_n<mode>" + [(match_operand:VMQ 0 "s_register_operand" "") + (match_operand:VMQ 1 "s_register_operand" "") + (match_operand:VMQ 2 "s_register_operand" "") + (match_operand:<V_elem> 3 "s_register_operand" "") + (match_operand:SI 4 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<V_HALF>mode); + emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx)); + emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2], + tmp, const0_rtx, operands[4])); + DONE; +}) + +(define_expand "neon_vmlsl_n<mode>" + [(match_operand:<V_widen> 0 "s_register_operand" "") + (match_operand:<V_widen> 1 "s_register_operand" "") + (match_operand:VMDI 2 "s_register_operand" "") + (match_operand:<V_elem> 3 "s_register_operand" "") + (match_operand:SI 4 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<MODE>mode); + emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx)); + emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2], + tmp, const0_rtx, operands[4])); + DONE; +}) + +(define_expand "neon_vqdmlsl_n<mode>" + [(match_operand:<V_widen> 0 "s_register_operand" "") + (match_operand:<V_widen> 1 "s_register_operand" "") + (match_operand:VMDI 2 "s_register_operand" "") + (match_operand:<V_elem> 3 "s_register_operand" "") + (match_operand:SI 4 "immediate_operand" "")] + "TARGET_NEON" +{ + rtx tmp = gen_reg_rtx (<MODE>mode); + emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx)); + emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2], + tmp, const0_rtx, operands[4])); + DONE; +}) + +(define_insn "neon_vext<mode>" + [(set (match_operand:VDQX 0 "s_register_operand" "=w") + (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w") + (match_operand:VDQX 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VEXT))] + "TARGET_NEON" +{ + neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); + return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_bp_simple") + (const_string "neon_bp_2cycle")))] +) + +(define_insn "neon_vrev64<mode>" + [(set (match_operand:VDQ 0 "s_register_operand" "=w") + (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VREV64))] + "TARGET_NEON" + "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1" + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vrev32<mode>" + [(set (match_operand:VX 0 "s_register_operand" "=w") + (unspec:VX [(match_operand:VX 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VREV32))] + "TARGET_NEON" + "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1" + [(set_attr "neon_type" "neon_bp_simple")] +) + +(define_insn "neon_vrev16<mode>" + [(set (match_operand:VE 0 "s_register_operand" "=w") + (unspec:VE [(match_operand:VE 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i")] + UNSPEC_VREV16))] + "TARGET_NEON" + "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1" + [(set_attr "neon_type" "neon_bp_simple")] +) + +; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register +; allocation. For an intrinsic of form: +; rD = vbsl_* (rS, rN, rM) +; We can use any of: +; vbsl rS, rN, rM (if D = S) +; vbit rD, rN, rS (if D = M, so 1-bits in rS choose bits from rN, else rM) +; vbif rD, rM, rS (if D = N, so 0-bits in rS choose bits from rM, else rN) + +(define_insn "neon_vbsl<mode>_internal" + [(set (match_operand:VDQX 0 "s_register_operand" "=w,w,w") + (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w") + (match_operand:VDQX 2 "s_register_operand" " w,w,0") + (match_operand:VDQX 3 "s_register_operand" " w,0,w")] + UNSPEC_VBSL))] + "TARGET_NEON" + "@ + vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3 + vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1 + vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1" + [(set_attr "neon_type" "neon_int_1")] +) + +(define_expand "neon_vbsl<mode>" + [(set (match_operand:VDQX 0 "s_register_operand" "") + (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "") + (match_operand:VDQX 2 "s_register_operand" "") + (match_operand:VDQX 3 "s_register_operand" "")] + UNSPEC_VBSL))] + "TARGET_NEON" +{ + /* We can't alias operands together if they have different modes. */ + operands[1] = gen_lowpart (<MODE>mode, operands[1]); +}) + +(define_insn "neon_vshl<mode>" + [(set (match_operand:VDQIX 0 "s_register_operand" "=w") + (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") + (match_operand:VDQIX 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VSHL))] + "TARGET_NEON" + "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_vshl_ddd") + (const_string "neon_shift_3")))] +) + +(define_insn "neon_vqshl<mode>" + [(set (match_operand:VDQIX 0 "s_register_operand" "=w") + (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") + (match_operand:VDQIX 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VQSHL))] + "TARGET_NEON" + "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_shift_2") + (const_string "neon_vqshl_vrshl_vqrshl_qqq")))] +) + +(define_insn "neon_vshr_n<mode>" + [(set (match_operand:VDQIX 0 "s_register_operand" "=w") + (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VSHR_N))] + "TARGET_NEON" +{ + neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1); + return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2"; +} + [(set_attr "neon_type" "neon_shift_1")] +) + +(define_insn "neon_vshrn_n<mode>" + [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") + (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VSHRN_N))] + "TARGET_NEON" +{ + neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1); + return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2"; +} + [(set_attr "neon_type" "neon_shift_1")] +) + +(define_insn "neon_vqshrn_n<mode>" + [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") + (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VQSHRN_N))] + "TARGET_NEON" +{ + neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1); + return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2"; +} + [(set_attr "neon_type" "neon_shift_2")] +) + +(define_insn "neon_vqshrun_n<mode>" + [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") + (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VQSHRUN_N))] + "TARGET_NEON" +{ + neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1); + return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2"; +} + [(set_attr "neon_type" "neon_shift_2")] +) + +(define_insn "neon_vshl_n<mode>" + [(set (match_operand:VDQIX 0 "s_register_operand" "=w") + (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VSHL_N))] + "TARGET_NEON" +{ + neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode)); + return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2"; +} + [(set_attr "neon_type" "neon_shift_1")] +) + +(define_insn "neon_vqshl_n<mode>" + [(set (match_operand:VDQIX 0 "s_register_operand" "=w") + (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VQSHL_N))] + "TARGET_NEON" +{ + neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode)); + return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2"; +} + [(set_attr "neon_type" "neon_shift_2")] +) + +(define_insn "neon_vqshlu_n<mode>" + [(set (match_operand:VDQIX 0 "s_register_operand" "=w") + (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VQSHLU_N))] + "TARGET_NEON" +{ + neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode)); + return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2"; +} + [(set_attr "neon_type" "neon_shift_2")] +) + +(define_insn "neon_vshll_n<mode>" + [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") + (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VSHLL_N))] + "TARGET_NEON" +{ + /* The boundaries are: 0 < imm <= size. */ + neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1); + return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2"; +} + [(set_attr "neon_type" "neon_shift_1")] +) + +(define_insn "neon_vsra_n<mode>" + [(set (match_operand:VDQIX 0 "s_register_operand" "=w") + (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0") + (match_operand:VDQIX 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i") + (match_operand:SI 4 "immediate_operand" "i")] + UNSPEC_VSRA_N))] + "TARGET_NEON" +{ + neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1); + return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3"; +} + [(set_attr "neon_type" "neon_vsra_vrsra")] +) + +(define_insn "neon_vsri_n<mode>" + [(set (match_operand:VDQIX 0 "s_register_operand" "=w") + (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0") + (match_operand:VDQIX 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VSRI))] + "TARGET_NEON" +{ + neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1); + return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_shift_1") + (const_string "neon_shift_3")))] +) + +(define_insn "neon_vsli_n<mode>" + [(set (match_operand:VDQIX 0 "s_register_operand" "=w") + (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0") + (match_operand:VDQIX 2 "s_register_operand" "w") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VSLI))] + "TARGET_NEON" +{ + neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode)); + return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3"; +} + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_shift_1") + (const_string "neon_shift_3")))] +) + +(define_insn "neon_vtbl1v8qi" + [(set (match_operand:V8QI 0 "s_register_operand" "=w") + (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w") + (match_operand:V8QI 2 "s_register_operand" "w")] + UNSPEC_VTBL))] + "TARGET_NEON" + "vtbl.8\t%P0, {%P1}, %P2" + [(set_attr "neon_type" "neon_bp_2cycle")] +) + +(define_insn "neon_vtbl2v8qi" + [(set (match_operand:V8QI 0 "s_register_operand" "=w") + (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w") + (match_operand:V8QI 2 "s_register_operand" "w")] + UNSPEC_VTBL))] + "TARGET_NEON" +{ + rtx ops[4]; + int tabbase = REGNO (operands[1]); + + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (V8QImode, tabbase); + ops[2] = gen_rtx_REG (V8QImode, tabbase + 2); + ops[3] = operands[2]; + output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops); + + return ""; +} + [(set_attr "neon_type" "neon_bp_2cycle")] +) + +(define_insn "neon_vtbl3v8qi" + [(set (match_operand:V8QI 0 "s_register_operand" "=w") + (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w") + (match_operand:V8QI 2 "s_register_operand" "w")] + UNSPEC_VTBL))] + "TARGET_NEON" +{ + rtx ops[5]; + int tabbase = REGNO (operands[1]); + + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (V8QImode, tabbase); + ops[2] = gen_rtx_REG (V8QImode, tabbase + 2); + ops[3] = gen_rtx_REG (V8QImode, tabbase + 4); + ops[4] = operands[2]; + output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops); + + return ""; +} + [(set_attr "neon_type" "neon_bp_3cycle")] +) + +(define_insn "neon_vtbl4v8qi" + [(set (match_operand:V8QI 0 "s_register_operand" "=w") + (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w") + (match_operand:V8QI 2 "s_register_operand" "w")] + UNSPEC_VTBL))] + "TARGET_NEON" +{ + rtx ops[6]; + int tabbase = REGNO (operands[1]); + + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (V8QImode, tabbase); + ops[2] = gen_rtx_REG (V8QImode, tabbase + 2); + ops[3] = gen_rtx_REG (V8QImode, tabbase + 4); + ops[4] = gen_rtx_REG (V8QImode, tabbase + 6); + ops[5] = operands[2]; + output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops); + + return ""; +} + [(set_attr "neon_type" "neon_bp_3cycle")] +) + +(define_insn "neon_vtbx1v8qi" + [(set (match_operand:V8QI 0 "s_register_operand" "=w") + (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0") + (match_operand:V8QI 2 "s_register_operand" "w") + (match_operand:V8QI 3 "s_register_operand" "w")] + UNSPEC_VTBX))] + "TARGET_NEON" + "vtbx.8\t%P0, {%P2}, %P3" + [(set_attr "neon_type" "neon_bp_2cycle")] +) + +(define_insn "neon_vtbx2v8qi" + [(set (match_operand:V8QI 0 "s_register_operand" "=w") + (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0") + (match_operand:TI 2 "s_register_operand" "w") + (match_operand:V8QI 3 "s_register_operand" "w")] + UNSPEC_VTBX))] + "TARGET_NEON" +{ + rtx ops[4]; + int tabbase = REGNO (operands[2]); + + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (V8QImode, tabbase); + ops[2] = gen_rtx_REG (V8QImode, tabbase + 2); + ops[3] = operands[3]; + output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops); + + return ""; +} + [(set_attr "neon_type" "neon_bp_2cycle")] +) + +(define_insn "neon_vtbx3v8qi" + [(set (match_operand:V8QI 0 "s_register_operand" "=w") + (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0") + (match_operand:EI 2 "s_register_operand" "w") + (match_operand:V8QI 3 "s_register_operand" "w")] + UNSPEC_VTBX))] + "TARGET_NEON" +{ + rtx ops[5]; + int tabbase = REGNO (operands[2]); + + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (V8QImode, tabbase); + ops[2] = gen_rtx_REG (V8QImode, tabbase + 2); + ops[3] = gen_rtx_REG (V8QImode, tabbase + 4); + ops[4] = operands[3]; + output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops); + + return ""; +} + [(set_attr "neon_type" "neon_bp_3cycle")] +) + +(define_insn "neon_vtbx4v8qi" + [(set (match_operand:V8QI 0 "s_register_operand" "=w") + (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0") + (match_operand:OI 2 "s_register_operand" "w") + (match_operand:V8QI 3 "s_register_operand" "w")] + UNSPEC_VTBX))] + "TARGET_NEON" +{ + rtx ops[6]; + int tabbase = REGNO (operands[2]); + + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (V8QImode, tabbase); + ops[2] = gen_rtx_REG (V8QImode, tabbase + 2); + ops[3] = gen_rtx_REG (V8QImode, tabbase + 4); + ops[4] = gen_rtx_REG (V8QImode, tabbase + 6); + ops[5] = operands[3]; + output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops); + + return ""; +} + [(set_attr "neon_type" "neon_bp_3cycle")] +) + +(define_insn "neon_vtrn<mode>_internal" + [(set (match_operand:VDQW 0 "s_register_operand" "=w") + (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0") + (match_operand:VDQW 2 "s_register_operand" "w")] + UNSPEC_VTRN1)) + (set (match_operand:VDQW 3 "s_register_operand" "=2") + (unspec:VDQW [(match_dup 1) (match_dup 2)] + UNSPEC_VTRN2))] + "TARGET_NEON" + "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>3" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_bp_simple") + (const_string "neon_bp_3cycle")))] +) + +(define_expand "neon_vtrn<mode>" + [(match_operand:SI 0 "s_register_operand" "r") + (match_operand:VDQW 1 "s_register_operand" "w") + (match_operand:VDQW 2 "s_register_operand" "w")] + "TARGET_NEON" +{ + neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal, + operands[0], operands[1], operands[2]); + DONE; +}) + +(define_insn "neon_vzip<mode>_internal" + [(set (match_operand:VDQW 0 "s_register_operand" "=w") + (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0") + (match_operand:VDQW 2 "s_register_operand" "w")] + UNSPEC_VZIP1)) + (set (match_operand:VDQW 3 "s_register_operand" "=2") + (unspec:VDQW [(match_dup 1) (match_dup 2)] + UNSPEC_VZIP2))] + "TARGET_NEON" + "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>3" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_bp_simple") + (const_string "neon_bp_3cycle")))] +) + +(define_expand "neon_vzip<mode>" + [(match_operand:SI 0 "s_register_operand" "r") + (match_operand:VDQW 1 "s_register_operand" "w") + (match_operand:VDQW 2 "s_register_operand" "w")] + "TARGET_NEON" +{ + neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal, + operands[0], operands[1], operands[2]); + DONE; +}) + +(define_insn "neon_vuzp<mode>_internal" + [(set (match_operand:VDQW 0 "s_register_operand" "=w") + (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0") + (match_operand:VDQW 2 "s_register_operand" "w")] + UNSPEC_VUZP1)) + (set (match_operand:VDQW 3 "s_register_operand" "=2") + (unspec:VDQW [(match_dup 1) (match_dup 2)] + UNSPEC_VUZP2))] + "TARGET_NEON" + "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>3" + [(set (attr "neon_type") + (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) + (const_string "neon_bp_simple") + (const_string "neon_bp_3cycle")))] +) + +(define_expand "neon_vuzp<mode>" + [(match_operand:SI 0 "s_register_operand" "r") + (match_operand:VDQW 1 "s_register_operand" "w") + (match_operand:VDQW 2 "s_register_operand" "w")] + "TARGET_NEON" +{ + neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal, + operands[0], operands[1], operands[2]); + DONE; +}) + +(define_expand "neon_vreinterpretv8qi<mode>" + [(match_operand:V8QI 0 "s_register_operand" "") + (match_operand:VDX 1 "s_register_operand" "")] + "TARGET_NEON" +{ + neon_reinterpret (operands[0], operands[1]); + DONE; +}) + +(define_expand "neon_vreinterpretv4hi<mode>" + [(match_operand:V4HI 0 "s_register_operand" "") + (match_operand:VDX 1 "s_register_operand" "")] + "TARGET_NEON" +{ + neon_reinterpret (operands[0], operands[1]); + DONE; +}) + +(define_expand "neon_vreinterpretv2si<mode>" + [(match_operand:V2SI 0 "s_register_operand" "") + (match_operand:VDX 1 "s_register_operand" "")] + "TARGET_NEON" +{ + neon_reinterpret (operands[0], operands[1]); + DONE; +}) + +(define_expand "neon_vreinterpretv2sf<mode>" + [(match_operand:V2SF 0 "s_register_operand" "") + (match_operand:VDX 1 "s_register_operand" "")] + "TARGET_NEON" +{ + neon_reinterpret (operands[0], operands[1]); + DONE; +}) + +(define_expand "neon_vreinterpretdi<mode>" + [(match_operand:DI 0 "s_register_operand" "") + (match_operand:VDX 1 "s_register_operand" "")] + "TARGET_NEON" +{ + neon_reinterpret (operands[0], operands[1]); + DONE; +}) + +(define_expand "neon_vreinterpretv16qi<mode>" + [(match_operand:V16QI 0 "s_register_operand" "") + (match_operand:VQX 1 "s_register_operand" "")] + "TARGET_NEON" +{ + neon_reinterpret (operands[0], operands[1]); + DONE; +}) + +(define_expand "neon_vreinterpretv8hi<mode>" + [(match_operand:V8HI 0 "s_register_operand" "") + (match_operand:VQX 1 "s_register_operand" "")] + "TARGET_NEON" +{ + neon_reinterpret (operands[0], operands[1]); + DONE; +}) + +(define_expand "neon_vreinterpretv4si<mode>" + [(match_operand:V4SI 0 "s_register_operand" "") + (match_operand:VQX 1 "s_register_operand" "")] + "TARGET_NEON" +{ + neon_reinterpret (operands[0], operands[1]); + DONE; +}) + +(define_expand "neon_vreinterpretv4sf<mode>" + [(match_operand:V4SF 0 "s_register_operand" "") + (match_operand:VQX 1 "s_register_operand" "")] + "TARGET_NEON" +{ + neon_reinterpret (operands[0], operands[1]); + DONE; +}) + +(define_expand "neon_vreinterpretv2di<mode>" + [(match_operand:V2DI 0 "s_register_operand" "") + (match_operand:VQX 1 "s_register_operand" "")] + "TARGET_NEON" +{ + neon_reinterpret (operands[0], operands[1]); + DONE; +}) + +(define_insn "neon_vld1<mode>" + [(set (match_operand:VDQX 0 "s_register_operand" "=w") + (unspec:VDQX [(mem:VDQX (match_operand:SI 1 "s_register_operand" "r"))] + UNSPEC_VLD1))] + "TARGET_NEON" + "vld1.<V_sz_elem>\t%h0, [%1]" + [(set_attr "neon_type" "neon_vld1_1_2_regs")] +) + +(define_insn "neon_vld1_lane<mode>" + [(set (match_operand:VDX 0 "s_register_operand" "=w") + (unspec:VDX [(mem:<V_elem> (match_operand:SI 1 "s_register_operand" "r")) + (match_operand:VDX 2 "s_register_operand" "0") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VLD1_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[3]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + if (lane < 0 || lane >= max) + error ("lane out of range"); + if (max == 1) + return "vld1.<V_sz_elem>\t%P0, [%1]"; + else + return "vld1.<V_sz_elem>\t{%P0[%c3]}, [%1]"; +} + [(set (attr "neon_type") + (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2)) + (const_string "neon_vld1_1_2_regs") + (const_string "neon_vld1_vld2_lane")))] +) + +(define_insn "neon_vld1_lane<mode>" + [(set (match_operand:VQX 0 "s_register_operand" "=w") + (unspec:VQX [(mem:<V_elem> (match_operand:SI 1 "s_register_operand" "r")) + (match_operand:VQX 2 "s_register_operand" "0") + (match_operand:SI 3 "immediate_operand" "i")] + UNSPEC_VLD1_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[3]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[0]); + if (lane < 0 || lane >= max) + error ("lane out of range"); + else if (lane >= max / 2) + { + lane -= max / 2; + regno += 2; + operands[3] = GEN_INT (lane); + } + operands[0] = gen_rtx_REG (<V_HALF>mode, regno); + if (max == 2) + return "vld1.<V_sz_elem>\t%P0, [%1]"; + else + return "vld1.<V_sz_elem>\t{%P0[%c3]}, [%1]"; +} + [(set (attr "neon_type") + (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2)) + (const_string "neon_vld1_1_2_regs") + (const_string "neon_vld1_vld2_lane")))] +) + +(define_insn "neon_vld1_dup<mode>" + [(set (match_operand:VDX 0 "s_register_operand" "=w") + (unspec:VDX [(mem:<V_elem> (match_operand:SI 1 "s_register_operand" "r"))] + UNSPEC_VLD1_DUP))] + "TARGET_NEON" +{ + if (GET_MODE_NUNITS (<MODE>mode) > 1) + return "vld1.<V_sz_elem>\t{%P0[]}, [%1]"; + else + return "vld1.<V_sz_elem>\t%h0, [%1]"; +} + [(set (attr "neon_type") + (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1")) + (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes") + (const_string "neon_vld1_1_2_regs")))] +) + +(define_insn "neon_vld1_dup<mode>" + [(set (match_operand:VQX 0 "s_register_operand" "=w") + (unspec:VQX [(mem:<V_elem> (match_operand:SI 1 "s_register_operand" "r"))] + UNSPEC_VLD1_DUP))] + "TARGET_NEON" +{ + if (GET_MODE_NUNITS (<MODE>mode) > 2) + return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, [%1]"; + else + return "vld1.<V_sz_elem>\t%h0, [%1]"; +} + [(set (attr "neon_type") + (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1")) + (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes") + (const_string "neon_vld1_1_2_regs")))] +) + +(define_insn "neon_vst1<mode>" + [(set (mem:VDQX (match_operand:SI 0 "s_register_operand" "r")) + (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")] + UNSPEC_VST1))] + "TARGET_NEON" + "vst1.<V_sz_elem>\t%h1, [%0]" + [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]) + +(define_insn "neon_vst1_lane<mode>" + [(set (mem:<V_elem> (match_operand:SI 0 "s_register_operand" "r")) + (vec_select:<V_elem> + (match_operand:VDX 1 "s_register_operand" "w") + (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[2]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + if (lane < 0 || lane >= max) + error ("lane out of range"); + if (max == 1) + return "vst1.<V_sz_elem>\t{%P1}, [%0]"; + else + return "vst1.<V_sz_elem>\t{%P1[%c2]}, [%0]"; +} + [(set (attr "neon_type") + (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 1)) + (const_string "neon_vst1_1_2_regs_vst2_2_regs") + (const_string "neon_vst1_vst2_lane")))]) + +(define_insn "neon_vst1_lane<mode>" + [(set (mem:<V_elem> (match_operand:SI 0 "s_register_operand" "r")) + (vec_select:<V_elem> + (match_operand:VQX 1 "s_register_operand" "w") + (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[2]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[1]); + if (lane < 0 || lane >= max) + error ("lane out of range"); + else if (lane >= max / 2) + { + lane -= max / 2; + regno += 2; + operands[2] = GEN_INT (lane); + } + operands[1] = gen_rtx_REG (<V_HALF>mode, regno); + if (max == 2) + return "vst1.<V_sz_elem>\t{%P1}, [%0]"; + else + return "vst1.<V_sz_elem>\t{%P1[%c2]}, [%0]"; +} + [(set_attr "neon_type" "neon_vst1_vst2_lane")] +) + +(define_insn "neon_vld2<mode>" + [(set (match_operand:TI 0 "s_register_operand" "=w") + (unspec:TI [(mem:TI (match_operand:SI 1 "s_register_operand" "r")) + (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD2))] + "TARGET_NEON" +{ + if (<V_sz_elem> == 64) + return "vld1.64\t%h0, [%1]"; + else + return "vld2.<V_sz_elem>\t%h0, [%1]"; +} + [(set (attr "neon_type") + (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64")) + (const_string "neon_vld1_1_2_regs") + (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")))] +) + +(define_insn "neon_vld2<mode>" + [(set (match_operand:OI 0 "s_register_operand" "=w") + (unspec:OI [(mem:OI (match_operand:SI 1 "s_register_operand" "r")) + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD2))] + "TARGET_NEON" + "vld2.<V_sz_elem>\t%h0, [%1]" + [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]) + +(define_insn "neon_vld2_lane<mode>" + [(set (match_operand:TI 0 "s_register_operand" "=w") + (unspec:TI [(mem:<V_two_elem> (match_operand:SI 1 "s_register_operand" "r")) + (match_operand:TI 2 "s_register_operand" "0") + (match_operand:SI 3 "immediate_operand" "i") + (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD2_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[3]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[0]); + rtx ops[4]; + if (lane < 0 || lane >= max) + error ("lane out of range"); + ops[0] = gen_rtx_REG (DImode, regno); + ops[1] = gen_rtx_REG (DImode, regno + 2); + ops[2] = operands[1]; + ops[3] = operands[3]; + output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, [%2]", ops); + return ""; +} + [(set_attr "neon_type" "neon_vld1_vld2_lane")] +) + +(define_insn "neon_vld2_lane<mode>" + [(set (match_operand:OI 0 "s_register_operand" "=w") + (unspec:OI [(mem:<V_two_elem> (match_operand:SI 1 "s_register_operand" "r")) + (match_operand:OI 2 "s_register_operand" "0") + (match_operand:SI 3 "immediate_operand" "i") + (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD2_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[3]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[0]); + rtx ops[4]; + if (lane < 0 || lane >= max) + error ("lane out of range"); + else if (lane >= max / 2) + { + lane -= max / 2; + regno += 2; + } + ops[0] = gen_rtx_REG (DImode, regno); + ops[1] = gen_rtx_REG (DImode, regno + 4); + ops[2] = operands[1]; + ops[3] = GEN_INT (lane); + output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, [%2]", ops); + return ""; +} + [(set_attr "neon_type" "neon_vld1_vld2_lane")] +) + +(define_insn "neon_vld2_dup<mode>" + [(set (match_operand:TI 0 "s_register_operand" "=w") + (unspec:TI [(mem:<V_two_elem> (match_operand:SI 1 "s_register_operand" "r")) + (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD2_DUP))] + "TARGET_NEON" +{ + if (GET_MODE_NUNITS (<MODE>mode) > 1) + return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, [%1]"; + else + return "vld1.<V_sz_elem>\t%h0, [%1]"; +} + [(set (attr "neon_type") + (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1")) + (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes") + (const_string "neon_vld1_1_2_regs")))] +) + +(define_insn "neon_vst2<mode>" + [(set (mem:TI (match_operand:SI 0 "s_register_operand" "r")) + (unspec:TI [(match_operand:TI 1 "s_register_operand" "w") + (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST2))] + "TARGET_NEON" +{ + if (<V_sz_elem> == 64) + return "vst1.64\t%h1, [%0]"; + else + return "vst2.<V_sz_elem>\t%h1, [%0]"; +} + [(set (attr "neon_type") + (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64")) + (const_string "neon_vst1_1_2_regs_vst2_2_regs") + (const_string "neon_vst1_1_2_regs_vst2_2_regs")))] +) + +(define_insn "neon_vst2<mode>" + [(set (mem:OI (match_operand:SI 0 "s_register_operand" "r")) + (unspec:OI [(match_operand:OI 1 "s_register_operand" "w") + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST2))] + "TARGET_NEON" + "vst2.<V_sz_elem>\t%h1, [%0]" + [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")] +) + +(define_insn "neon_vst2_lane<mode>" + [(set (mem:<V_two_elem> (match_operand:SI 0 "s_register_operand" "r")) + (unspec:<V_two_elem> + [(match_operand:TI 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST2_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[2]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[1]); + rtx ops[4]; + if (lane < 0 || lane >= max) + error ("lane out of range"); + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (DImode, regno); + ops[2] = gen_rtx_REG (DImode, regno + 2); + ops[3] = operands[2]; + output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, [%0]", ops); + return ""; +} + [(set_attr "neon_type" "neon_vst1_vst2_lane")] +) + +(define_insn "neon_vst2_lane<mode>" + [(set (mem:<V_two_elem> (match_operand:SI 0 "s_register_operand" "r")) + (unspec:<V_two_elem> + [(match_operand:OI 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST2_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[2]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[1]); + rtx ops[4]; + if (lane < 0 || lane >= max) + error ("lane out of range"); + else if (lane >= max / 2) + { + lane -= max / 2; + regno += 2; + } + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (DImode, regno); + ops[2] = gen_rtx_REG (DImode, regno + 4); + ops[3] = GEN_INT (lane); + output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, [%0]", ops); + return ""; +} + [(set_attr "neon_type" "neon_vst1_vst2_lane")] +) + +(define_insn "neon_vld3<mode>" + [(set (match_operand:EI 0 "s_register_operand" "=w") + (unspec:EI [(mem:EI (match_operand:SI 1 "s_register_operand" "r")) + (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD3))] + "TARGET_NEON" +{ + if (<V_sz_elem> == 64) + return "vld1.64\t%h0, [%1]"; + else + return "vld3.<V_sz_elem>\t%h0, [%1]"; +} + [(set (attr "neon_type") + (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64")) + (const_string "neon_vld1_1_2_regs") + (const_string "neon_vld3_vld4")))] +) + +(define_expand "neon_vld3<mode>" + [(match_operand:CI 0 "s_register_operand" "=w") + (match_operand:SI 1 "s_register_operand" "+r") + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + "TARGET_NEON" +{ + emit_insn (gen_neon_vld3qa<mode> (operands[0], operands[0], + operands[1], operands[1])); + emit_insn (gen_neon_vld3qb<mode> (operands[0], operands[0], + operands[1], operands[1])); + DONE; +}) + +(define_insn "neon_vld3qa<mode>" + [(set (match_operand:CI 0 "s_register_operand" "=w") + (unspec:CI [(mem:CI (match_operand:SI 3 "s_register_operand" "2")) + (match_operand:CI 1 "s_register_operand" "0") + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD3A)) + (set (match_operand:SI 2 "s_register_operand" "=r") + (plus:SI (match_dup 3) + (const_int 24)))] + "TARGET_NEON" +{ + int regno = REGNO (operands[0]); + rtx ops[4]; + ops[0] = gen_rtx_REG (DImode, regno); + ops[1] = gen_rtx_REG (DImode, regno + 4); + ops[2] = gen_rtx_REG (DImode, regno + 8); + ops[3] = operands[2]; + output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, [%3]!", ops); + return ""; +} + [(set_attr "neon_type" "neon_vld3_vld4")] +) + +(define_insn "neon_vld3qb<mode>" + [(set (match_operand:CI 0 "s_register_operand" "=w") + (unspec:CI [(mem:CI (match_operand:SI 3 "s_register_operand" "2")) + (match_operand:CI 1 "s_register_operand" "0") + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD3B)) + (set (match_operand:SI 2 "s_register_operand" "=r") + (plus:SI (match_dup 3) + (const_int 24)))] + "TARGET_NEON" +{ + int regno = REGNO (operands[0]); + rtx ops[4]; + ops[0] = gen_rtx_REG (DImode, regno + 2); + ops[1] = gen_rtx_REG (DImode, regno + 6); + ops[2] = gen_rtx_REG (DImode, regno + 10); + ops[3] = operands[2]; + output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, [%3]!", ops); + return ""; +} + [(set_attr "neon_type" "neon_vld3_vld4")] +) + +(define_insn "neon_vld3_lane<mode>" + [(set (match_operand:EI 0 "s_register_operand" "=w") + (unspec:EI [(mem:<V_three_elem> (match_operand:SI 1 "s_register_operand" "r")) + (match_operand:EI 2 "s_register_operand" "0") + (match_operand:SI 3 "immediate_operand" "i") + (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD3_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[3]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[0]); + rtx ops[5]; + if (lane < 0 || lane >= max) + error ("lane out of range"); + ops[0] = gen_rtx_REG (DImode, regno); + ops[1] = gen_rtx_REG (DImode, regno + 2); + ops[2] = gen_rtx_REG (DImode, regno + 4); + ops[3] = operands[1]; + ops[4] = operands[3]; + output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, [%3]", + ops); + return ""; +} + [(set_attr "neon_type" "neon_vld3_vld4_lane")] +) + +(define_insn "neon_vld3_lane<mode>" + [(set (match_operand:CI 0 "s_register_operand" "=w") + (unspec:CI [(mem:<V_three_elem> (match_operand:SI 1 "s_register_operand" "r")) + (match_operand:CI 2 "s_register_operand" "0") + (match_operand:SI 3 "immediate_operand" "i") + (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD3_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[3]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[0]); + rtx ops[5]; + if (lane < 0 || lane >= max) + error ("lane out of range"); + else if (lane >= max / 2) + { + lane -= max / 2; + regno += 2; + } + ops[0] = gen_rtx_REG (DImode, regno); + ops[1] = gen_rtx_REG (DImode, regno + 4); + ops[2] = gen_rtx_REG (DImode, regno + 8); + ops[3] = operands[1]; + ops[4] = GEN_INT (lane); + output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, [%3]", + ops); + return ""; +} + [(set_attr "neon_type" "neon_vld3_vld4_lane")] +) + +(define_insn "neon_vld3_dup<mode>" + [(set (match_operand:EI 0 "s_register_operand" "=w") + (unspec:EI [(mem:<V_three_elem> (match_operand:SI 1 "s_register_operand" "r")) + (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD3_DUP))] + "TARGET_NEON" +{ + if (GET_MODE_NUNITS (<MODE>mode) > 1) + { + int regno = REGNO (operands[0]); + rtx ops[4]; + ops[0] = gen_rtx_REG (DImode, regno); + ops[1] = gen_rtx_REG (DImode, regno + 2); + ops[2] = gen_rtx_REG (DImode, regno + 4); + ops[3] = operands[1]; + output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, [%3]", ops); + return ""; + } + else + return "vld1.<V_sz_elem>\t%h0, [%1]"; +} + [(set (attr "neon_type") + (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1")) + (const_string "neon_vld3_vld4_all_lanes") + (const_string "neon_vld1_1_2_regs")))]) + +(define_insn "neon_vst3<mode>" + [(set (mem:EI (match_operand:SI 0 "s_register_operand" "r")) + (unspec:EI [(match_operand:EI 1 "s_register_operand" "w") + (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST3))] + "TARGET_NEON" +{ + if (<V_sz_elem> == 64) + return "vst1.64\t%h1, [%0]"; + else + return "vst3.<V_sz_elem>\t%h1, [%0]"; +} + [(set (attr "neon_type") + (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64")) + (const_string "neon_vst1_1_2_regs_vst2_2_regs") + (const_string "neon_vst2_4_regs_vst3_vst4")))]) + +(define_expand "neon_vst3<mode>" + [(match_operand:SI 0 "s_register_operand" "+r") + (match_operand:CI 1 "s_register_operand" "w") + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + "TARGET_NEON" +{ + emit_insn (gen_neon_vst3qa<mode> (operands[0], operands[0], operands[1])); + emit_insn (gen_neon_vst3qb<mode> (operands[0], operands[0], operands[1])); + DONE; +}) + +(define_insn "neon_vst3qa<mode>" + [(set (mem:EI (match_operand:SI 1 "s_register_operand" "0")) + (unspec:EI [(match_operand:CI 2 "s_register_operand" "w") + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST3A)) + (set (match_operand:SI 0 "s_register_operand" "=r") + (plus:SI (match_dup 1) + (const_int 24)))] + "TARGET_NEON" +{ + int regno = REGNO (operands[2]); + rtx ops[4]; + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (DImode, regno); + ops[2] = gen_rtx_REG (DImode, regno + 4); + ops[3] = gen_rtx_REG (DImode, regno + 8); + output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, [%0]!", ops); + return ""; +} + [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")] +) + +(define_insn "neon_vst3qb<mode>" + [(set (mem:EI (match_operand:SI 1 "s_register_operand" "0")) + (unspec:EI [(match_operand:CI 2 "s_register_operand" "w") + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST3B)) + (set (match_operand:SI 0 "s_register_operand" "=r") + (plus:SI (match_dup 1) + (const_int 24)))] + "TARGET_NEON" +{ + int regno = REGNO (operands[2]); + rtx ops[4]; + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (DImode, regno + 2); + ops[2] = gen_rtx_REG (DImode, regno + 6); + ops[3] = gen_rtx_REG (DImode, regno + 10); + output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, [%0]!", ops); + return ""; +} + [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")] +) + +(define_insn "neon_vst3_lane<mode>" + [(set (mem:<V_three_elem> (match_operand:SI 0 "s_register_operand" "r")) + (unspec:<V_three_elem> + [(match_operand:EI 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST3_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[2]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[1]); + rtx ops[5]; + if (lane < 0 || lane >= max) + error ("lane out of range"); + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (DImode, regno); + ops[2] = gen_rtx_REG (DImode, regno + 2); + ops[3] = gen_rtx_REG (DImode, regno + 4); + ops[4] = operands[2]; + output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, [%0]", + ops); + return ""; +} + [(set_attr "neon_type" "neon_vst3_vst4_lane")] +) + +(define_insn "neon_vst3_lane<mode>" + [(set (mem:<V_three_elem> (match_operand:SI 0 "s_register_operand" "r")) + (unspec:<V_three_elem> + [(match_operand:CI 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST3_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[2]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[1]); + rtx ops[5]; + if (lane < 0 || lane >= max) + error ("lane out of range"); + else if (lane >= max / 2) + { + lane -= max / 2; + regno += 2; + } + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (DImode, regno); + ops[2] = gen_rtx_REG (DImode, regno + 4); + ops[3] = gen_rtx_REG (DImode, regno + 8); + ops[4] = GEN_INT (lane); + output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, [%0]", + ops); + return ""; +} +[(set_attr "neon_type" "neon_vst3_vst4_lane")]) + +(define_insn "neon_vld4<mode>" + [(set (match_operand:OI 0 "s_register_operand" "=w") + (unspec:OI [(mem:OI (match_operand:SI 1 "s_register_operand" "r")) + (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD4))] + "TARGET_NEON" +{ + if (<V_sz_elem> == 64) + return "vld1.64\t%h0, [%1]"; + else + return "vld4.<V_sz_elem>\t%h0, [%1]"; +} + [(set (attr "neon_type") + (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64")) + (const_string "neon_vld1_1_2_regs") + (const_string "neon_vld3_vld4")))] +) + +(define_expand "neon_vld4<mode>" + [(match_operand:XI 0 "s_register_operand" "=w") + (match_operand:SI 1 "s_register_operand" "+r") + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + "TARGET_NEON" +{ + emit_insn (gen_neon_vld4qa<mode> (operands[0], operands[0], + operands[1], operands[1])); + emit_insn (gen_neon_vld4qb<mode> (operands[0], operands[0], + operands[1], operands[1])); + DONE; +}) + +(define_insn "neon_vld4qa<mode>" + [(set (match_operand:XI 0 "s_register_operand" "=w") + (unspec:XI [(mem:XI (match_operand:SI 3 "s_register_operand" "2")) + (match_operand:XI 1 "s_register_operand" "0") + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD4A)) + (set (match_operand:SI 2 "s_register_operand" "=r") + (plus:SI (match_dup 3) + (const_int 32)))] + "TARGET_NEON" +{ + int regno = REGNO (operands[0]); + rtx ops[5]; + ops[0] = gen_rtx_REG (DImode, regno); + ops[1] = gen_rtx_REG (DImode, regno + 4); + ops[2] = gen_rtx_REG (DImode, regno + 8); + ops[3] = gen_rtx_REG (DImode, regno + 12); + ops[4] = operands[2]; + output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, [%4]!", ops); + return ""; +} + [(set_attr "neon_type" "neon_vld3_vld4")] +) + +(define_insn "neon_vld4qb<mode>" + [(set (match_operand:XI 0 "s_register_operand" "=w") + (unspec:XI [(mem:XI (match_operand:SI 3 "s_register_operand" "2")) + (match_operand:XI 1 "s_register_operand" "0") + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD4B)) + (set (match_operand:SI 2 "s_register_operand" "=r") + (plus:SI (match_dup 3) + (const_int 32)))] + "TARGET_NEON" +{ + int regno = REGNO (operands[0]); + rtx ops[5]; + ops[0] = gen_rtx_REG (DImode, regno + 2); + ops[1] = gen_rtx_REG (DImode, regno + 6); + ops[2] = gen_rtx_REG (DImode, regno + 10); + ops[3] = gen_rtx_REG (DImode, regno + 14); + ops[4] = operands[2]; + output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, [%4]!", ops); + return ""; +} + [(set_attr "neon_type" "neon_vld3_vld4")] +) + +(define_insn "neon_vld4_lane<mode>" + [(set (match_operand:OI 0 "s_register_operand" "=w") + (unspec:OI [(mem:<V_four_elem> (match_operand:SI 1 "s_register_operand" "r")) + (match_operand:OI 2 "s_register_operand" "0") + (match_operand:SI 3 "immediate_operand" "i") + (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD4_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[3]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[0]); + rtx ops[6]; + if (lane < 0 || lane >= max) + error ("lane out of range"); + ops[0] = gen_rtx_REG (DImode, regno); + ops[1] = gen_rtx_REG (DImode, regno + 2); + ops[2] = gen_rtx_REG (DImode, regno + 4); + ops[3] = gen_rtx_REG (DImode, regno + 6); + ops[4] = operands[1]; + ops[5] = operands[3]; + output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, [%4]", + ops); + return ""; +} + [(set_attr "neon_type" "neon_vld3_vld4_lane")] +) + +(define_insn "neon_vld4_lane<mode>" + [(set (match_operand:XI 0 "s_register_operand" "=w") + (unspec:XI [(mem:<V_four_elem> (match_operand:SI 1 "s_register_operand" "r")) + (match_operand:XI 2 "s_register_operand" "0") + (match_operand:SI 3 "immediate_operand" "i") + (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD4_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[3]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[0]); + rtx ops[6]; + if (lane < 0 || lane >= max) + error ("lane out of range"); + else if (lane >= max / 2) + { + lane -= max / 2; + regno += 2; + } + ops[0] = gen_rtx_REG (DImode, regno); + ops[1] = gen_rtx_REG (DImode, regno + 4); + ops[2] = gen_rtx_REG (DImode, regno + 8); + ops[3] = gen_rtx_REG (DImode, regno + 12); + ops[4] = operands[1]; + ops[5] = GEN_INT (lane); + output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, [%4]", + ops); + return ""; +} + [(set_attr "neon_type" "neon_vld3_vld4_lane")] +) + +(define_insn "neon_vld4_dup<mode>" + [(set (match_operand:OI 0 "s_register_operand" "=w") + (unspec:OI [(mem:<V_four_elem> (match_operand:SI 1 "s_register_operand" "r")) + (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VLD4_DUP))] + "TARGET_NEON" +{ + if (GET_MODE_NUNITS (<MODE>mode) > 1) + { + int regno = REGNO (operands[0]); + rtx ops[5]; + ops[0] = gen_rtx_REG (DImode, regno); + ops[1] = gen_rtx_REG (DImode, regno + 2); + ops[2] = gen_rtx_REG (DImode, regno + 4); + ops[3] = gen_rtx_REG (DImode, regno + 6); + ops[4] = operands[1]; + output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, [%4]", + ops); + return ""; + } + else + return "vld1.<V_sz_elem>\t%h0, [%1]"; +} + [(set (attr "neon_type") + (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1")) + (const_string "neon_vld3_vld4_all_lanes") + (const_string "neon_vld1_1_2_regs")))] +) + +(define_insn "neon_vst4<mode>" + [(set (mem:OI (match_operand:SI 0 "s_register_operand" "r")) + (unspec:OI [(match_operand:OI 1 "s_register_operand" "w") + (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST4))] + "TARGET_NEON" +{ + if (<V_sz_elem> == 64) + return "vst1.64\t%h1, [%0]"; + else + return "vst4.<V_sz_elem>\t%h1, [%0]"; +} + [(set (attr "neon_type") + (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64")) + (const_string "neon_vst1_1_2_regs_vst2_2_regs") + (const_string "neon_vst2_4_regs_vst3_vst4")))] +) + +(define_expand "neon_vst4<mode>" + [(match_operand:SI 0 "s_register_operand" "+r") + (match_operand:XI 1 "s_register_operand" "w") + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + "TARGET_NEON" +{ + emit_insn (gen_neon_vst4qa<mode> (operands[0], operands[0], operands[1])); + emit_insn (gen_neon_vst4qb<mode> (operands[0], operands[0], operands[1])); + DONE; +}) + +(define_insn "neon_vst4qa<mode>" + [(set (mem:OI (match_operand:SI 1 "s_register_operand" "0")) + (unspec:OI [(match_operand:XI 2 "s_register_operand" "w") + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST4A)) + (set (match_operand:SI 0 "s_register_operand" "=r") + (plus:SI (match_dup 1) + (const_int 32)))] + "TARGET_NEON" +{ + int regno = REGNO (operands[2]); + rtx ops[5]; + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (DImode, regno); + ops[2] = gen_rtx_REG (DImode, regno + 4); + ops[3] = gen_rtx_REG (DImode, regno + 8); + ops[4] = gen_rtx_REG (DImode, regno + 12); + output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, [%0]!", ops); + return ""; +} + [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")] +) + +(define_insn "neon_vst4qb<mode>" + [(set (mem:OI (match_operand:SI 1 "s_register_operand" "0")) + (unspec:OI [(match_operand:XI 2 "s_register_operand" "w") + (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST4B)) + (set (match_operand:SI 0 "s_register_operand" "=r") + (plus:SI (match_dup 1) + (const_int 32)))] + "TARGET_NEON" +{ + int regno = REGNO (operands[2]); + rtx ops[5]; + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (DImode, regno + 2); + ops[2] = gen_rtx_REG (DImode, regno + 6); + ops[3] = gen_rtx_REG (DImode, regno + 10); + ops[4] = gen_rtx_REG (DImode, regno + 14); + output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, [%0]!", ops); + return ""; +} + [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")] +) + +(define_insn "neon_vst4_lane<mode>" + [(set (mem:<V_four_elem> (match_operand:SI 0 "s_register_operand" "r")) + (unspec:<V_four_elem> + [(match_operand:OI 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST4_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[2]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[1]); + rtx ops[6]; + if (lane < 0 || lane >= max) + error ("lane out of range"); + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (DImode, regno); + ops[2] = gen_rtx_REG (DImode, regno + 2); + ops[3] = gen_rtx_REG (DImode, regno + 4); + ops[4] = gen_rtx_REG (DImode, regno + 6); + ops[5] = operands[2]; + output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, [%0]", + ops); + return ""; +} + [(set_attr "neon_type" "neon_vst3_vst4_lane")] +) + +(define_insn "neon_vst4_lane<mode>" + [(set (mem:<V_four_elem> (match_operand:SI 0 "s_register_operand" "r")) + (unspec:<V_four_elem> + [(match_operand:XI 1 "s_register_operand" "w") + (match_operand:SI 2 "immediate_operand" "i") + (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] + UNSPEC_VST4_LANE))] + "TARGET_NEON" +{ + HOST_WIDE_INT lane = INTVAL (operands[2]); + HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); + int regno = REGNO (operands[1]); + rtx ops[6]; + if (lane < 0 || lane >= max) + error ("lane out of range"); + else if (lane >= max / 2) + { + lane -= max / 2; + regno += 2; + } + ops[0] = operands[0]; + ops[1] = gen_rtx_REG (DImode, regno); + ops[2] = gen_rtx_REG (DImode, regno + 4); + ops[3] = gen_rtx_REG (DImode, regno + 8); + ops[4] = gen_rtx_REG (DImode, regno + 12); + ops[5] = GEN_INT (lane); + output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, [%0]", + ops); + return ""; +} + [(set_attr "neon_type" "neon_vst3_vst4_lane")] +) + +(define_expand "neon_vand<mode>" + [(match_operand:VDQX 0 "s_register_operand" "") + (match_operand:VDQX 1 "s_register_operand" "") + (match_operand:VDQX 2 "neon_inv_logic_op2" "") + (match_operand:SI 3 "immediate_operand" "")] + "TARGET_NEON" +{ + emit_insn (gen_and<mode>3<V_suf64> (operands[0], operands[1], operands[2])); + DONE; +}) + +(define_expand "neon_vorr<mode>" + [(match_operand:VDQX 0 "s_register_operand" "") + (match_operand:VDQX 1 "s_register_operand" "") + (match_operand:VDQX 2 "neon_logic_op2" "") + (match_operand:SI 3 "immediate_operand" "")] + "TARGET_NEON" +{ + emit_insn (gen_ior<mode>3<V_suf64> (operands[0], operands[1], operands[2])); + DONE; +}) + +(define_expand "neon_veor<mode>" + [(match_operand:VDQX 0 "s_register_operand" "") + (match_operand:VDQX 1 "s_register_operand" "") + (match_operand:VDQX 2 "s_register_operand" "") + (match_operand:SI 3 "immediate_operand" "")] + "TARGET_NEON" +{ + emit_insn (gen_xor<mode>3<V_suf64> (operands[0], operands[1], operands[2])); + DONE; +}) + +(define_expand "neon_vbic<mode>" + [(match_operand:VDQX 0 "s_register_operand" "") + (match_operand:VDQX 1 "s_register_operand" "") + (match_operand:VDQX 2 "neon_logic_op2" "") + (match_operand:SI 3 "immediate_operand" "")] + "TARGET_NEON" +{ + emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2])); + DONE; +}) + +(define_expand "neon_vorn<mode>" + [(match_operand:VDQX 0 "s_register_operand" "") + (match_operand:VDQX 1 "s_register_operand" "") + (match_operand:VDQX 2 "neon_inv_logic_op2" "") + (match_operand:SI 3 "immediate_operand" "")] + "TARGET_NEON" +{ + emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2])); + DONE; +}) + +(define_insn "neon_vec_unpack<US>_lo_<mode>" + [(set (match_operand:<V_unpack> 0 "register_operand" "=w") + (SE:<V_unpack> (vec_select:<V_HALF> + (match_operand:VU 1 "register_operand" "w") + (match_operand:VU 2 "vect_par_constant_low" ""))))] + "TARGET_NEON" + "vmovl.<US><V_sz_elem> %q0, %e1" + [(set_attr "neon_type" "neon_shift_1")] +) + +(define_insn "neon_vec_unpack<US>_hi_<mode>" + [(set (match_operand:<V_unpack> 0 "register_operand" "=w") + (SE:<V_unpack> (vec_select:<V_HALF> + (match_operand:VU 1 "register_operand" "w") + (match_operand:VU 2 "vect_par_constant_high" ""))))] + "TARGET_NEON" + "vmovl.<US><V_sz_elem> %q0, %f1" + [(set_attr "neon_type" "neon_shift_1")] +) + +(define_expand "vec_unpack<US>_hi_<mode>" + [(match_operand:<V_unpack> 0 "register_operand" "") + (SE:<V_unpack> (match_operand:VU 1 "register_operand"))] + "TARGET_NEON" + { + rtvec v = rtvec_alloc (<V_mode_nunits>/2) ; + rtx t1; + int i; + for (i = 0; i < (<V_mode_nunits>/2); i++) + RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i); + + t1 = gen_rtx_PARALLEL (<MODE>mode, v); + emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0], + operands[1], + t1)); + DONE; + } +) + +(define_expand "vec_unpack<US>_lo_<mode>" + [(match_operand:<V_unpack> 0 "register_operand" "") + (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))] + "TARGET_NEON" + { + rtvec v = rtvec_alloc (<V_mode_nunits>/2) ; + rtx t1; + int i; + for (i = 0; i < (<V_mode_nunits>/2) ; i++) + RTVEC_ELT (v, i) = GEN_INT (i); + t1 = gen_rtx_PARALLEL (<MODE>mode, v); + emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0], + operands[1], + t1)); + DONE; + } +) + +(define_insn "neon_vec_<US>mult_lo_<mode>" + [(set (match_operand:<V_unpack> 0 "register_operand" "=w") + (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF> + (match_operand:VU 1 "register_operand" "w") + (match_operand:VU 2 "vect_par_constant_low" ""))) + (SE:<V_unpack> (vec_select:<V_HALF> + (match_operand:VU 3 "register_operand" "w") + (match_dup 2)))))] + "TARGET_NEON" + "vmull.<US><V_sz_elem> %q0, %e1, %e3" + [(set_attr "neon_type" "neon_shift_1")] +) + +(define_expand "vec_widen_<US>mult_lo_<mode>" + [(match_operand:<V_unpack> 0 "register_operand" "") + (SE:<V_unpack> (match_operand:VU 1 "register_operand" "")) + (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))] + "TARGET_NEON" + { + rtvec v = rtvec_alloc (<V_mode_nunits>/2) ; + rtx t1; + int i; + for (i = 0; i < (<V_mode_nunits>/2) ; i++) + RTVEC_ELT (v, i) = GEN_INT (i); + t1 = gen_rtx_PARALLEL (<MODE>mode, v); + + emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0], + operands[1], + t1, + operands[2])); + DONE; + } +) + +(define_insn "neon_vec_<US>mult_hi_<mode>" + [(set (match_operand:<V_unpack> 0 "register_operand" "=w") + (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF> + (match_operand:VU 1 "register_operand" "w") + (match_operand:VU 2 "vect_par_constant_high" ""))) + (SE:<V_unpack> (vec_select:<V_HALF> + (match_operand:VU 3 "register_operand" "w") + (match_dup 2)))))] + "TARGET_NEON" + "vmull.<US><V_sz_elem> %q0, %f1, %f3" + [(set_attr "neon_type" "neon_shift_1")] +) + +(define_expand "vec_widen_<US>mult_hi_<mode>" + [(match_operand:<V_unpack> 0 "register_operand" "") + (SE:<V_unpack> (match_operand:VU 1 "register_operand" "")) + (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))] + "TARGET_NEON" + { + rtvec v = rtvec_alloc (<V_mode_nunits>/2) ; + rtx t1; + int i; + for (i = 0; i < (<V_mode_nunits>/2) ; i++) + RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i); + t1 = gen_rtx_PARALLEL (<MODE>mode, v); + + emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0], + operands[1], + t1, + operands[2])); + DONE; + + } +) + +;; Vectorize for non-neon-quad case +(define_insn "neon_unpack<US>_<mode>" + [(set (match_operand:<V_widen> 0 "register_operand" "=w") + (SE:<V_widen> (match_operand:VDI 1 "register_operand" "w")))] + "TARGET_NEON" + "vmovl.<US><V_sz_elem> %q0, %P1" + [(set_attr "neon_type" "neon_shift_1")] +) + +(define_expand "vec_unpack<US>_lo_<mode>" + [(match_operand:<V_double_width> 0 "register_operand" "") + (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))] + "TARGET_NEON" +{ + rtx tmpreg = gen_reg_rtx (<V_widen>mode); + emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1])); + emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg)); + + DONE; +} +) + +(define_expand "vec_unpack<US>_hi_<mode>" + [(match_operand:<V_double_width> 0 "register_operand" "") + (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))] + "TARGET_NEON" +{ + rtx tmpreg = gen_reg_rtx (<V_widen>mode); + emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1])); + emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg)); + + DONE; +} +) + +(define_insn "neon_vec_<US>mult_<mode>" + [(set (match_operand:<V_widen> 0 "register_operand" "=w") + (mult:<V_widen> (SE:<V_widen> + (match_operand:VDI 1 "register_operand" "w")) + (SE:<V_widen> + (match_operand:VDI 2 "register_operand" "w"))))] + "TARGET_NEON" + "vmull.<US><V_sz_elem> %q0, %P1, %P2" + [(set_attr "neon_type" "neon_shift_1")] +) + +(define_expand "vec_widen_<US>mult_hi_<mode>" + [(match_operand:<V_double_width> 0 "register_operand" "") + (SE:<V_double_width> (match_operand:VDI 1 "register_operand" "")) + (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))] + "TARGET_NEON" + { + rtx tmpreg = gen_reg_rtx (<V_widen>mode); + emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2])); + emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg)); + + DONE; + + } +) + +(define_expand "vec_widen_<US>mult_lo_<mode>" + [(match_operand:<V_double_width> 0 "register_operand" "") + (SE:<V_double_width> (match_operand:VDI 1 "register_operand" "")) + (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))] + "TARGET_NEON" + { + rtx tmpreg = gen_reg_rtx (<V_widen>mode); + emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2])); + emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg)); + + DONE; + + } +) + +;; The case when using all quad registers. +(define_insn "vec_pack_trunc_<mode>" + [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w") + (vec_concat:<V_narrow_pack> + (truncate:<V_narrow> + (match_operand:VN 1 "register_operand" "w")) + (truncate:<V_narrow> + (match_operand:VN 2 "register_operand" "w"))))] + "TARGET_NEON" + "vmovn.i<V_sz_elem>\t%e0, %q1\;vmovn.i<V_sz_elem>\t%f0, %q2" + [(set_attr "neon_type" "neon_shift_1") + (set_attr "length" "8")] +) + +;; For the non-quad case. +(define_insn "neon_vec_pack_trunc_<mode>" + [(set (match_operand:<V_narrow> 0 "register_operand" "=w") + (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "w")))] + "TARGET_NEON" + "vmovn.i<V_sz_elem>\t%P0, %q1" + [(set_attr "neon_type" "neon_shift_1")] +) + +(define_expand "vec_pack_trunc_<mode>" + [(match_operand:<V_narrow_pack> 0 "register_operand" "") + (match_operand:VSHFT 1 "register_operand" "") + (match_operand:VSHFT 2 "register_operand")] + "TARGET_NEON" +{ + rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode); + + emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1])); + emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2])); + emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg)); + DONE; +}) |